From b8c88d2f639f54f265b32499837f16643eeb0287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Wed, 22 Jun 2022 15:09:33 +0200 Subject: [PATCH 01/29] #20 - php-di and guzzle updates --- composer.json | 4 ++-- src/System/DependencyConfig.php | 38 +++++++++++++++++---------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/composer.json b/composer.json index 1e84430..995864c 100755 --- a/composer.json +++ b/composer.json @@ -3,11 +3,11 @@ "description": "A database production to sandbox utility to sanitize data.", "type": "library", "require": { - "php-di/php-di": "^5.4", + "php-di/php-di": "^6.0", "aws/aws-sdk-php": "^3.19", "symfony/yaml": ">=2.3", "ericpoe/haystack": "^1.0", - "guzzlehttp/guzzle": "^6.2", + "guzzlehttp/guzzle": "^6.2|^7.0", "psr/log": "^1.0", "symfony/console": "^4.4", "symfony/event-dispatcher": "^4.0" diff --git a/src/System/DependencyConfig.php b/src/System/DependencyConfig.php index 965e646..88f8c48 100644 --- a/src/System/DependencyConfig.php +++ b/src/System/DependencyConfig.php @@ -17,13 +17,14 @@ * @package default **/ +declare(strict_types=1); + namespace Driver\System; use DI; +use DI\Definition\Helper\DefinitionHelper; use Driver\Engines\LocalConnectionInterface; use Driver\Engines\MySql\Sandbox\Connection; -use Driver\Engines\MySql\Sandbox\Export; -use Driver\Engines\MySql\Sandbox\Import; use Driver\Engines\RemoteConnectionInterface; use Driver\Pipeline\Environment; use Driver\Pipeline\Stage; @@ -32,45 +33,46 @@ use Driver\Pipeline\Transport\Primary as TransportPrimary; use Driver\System\Logs\LoggerInterface; use Driver\System\Logs\Primary; -use Driver\System\DebugMode; -use Symfony\Component\Console\Application as ConsoleApplication; -use Symfony\Component\Console\Tester\ApplicationTester as ConsoleApplicationTester; class DependencyConfig { - private bool $isDebug = false; + private bool $isDebug; public function __construct(bool $isDebug) { $this->isDebug = $isDebug; } - public function get() + /** + * @return + */ + public function get(): array { - $output = [ + return [ LoggerInterface::class => DI\Factory(function() { return new Primary(); }), Environment\EnvironmentInterface::class => DI\factory([Environment\Primary::class, 'create']), - Environment\Factory::class => DI\object()->constructorParameter('type', Environment\Primary::class), + Environment\Factory::class => DI\autowire()->constructorParameter('type', Environment\Primary::class), Stage\StageInterface::class => DI\factory([Stage\Primary::class, 'create']), - Stage\Factory::class => DI\object()->constructorParameter('type', Stage\Primary::class), + Stage\Factory::class => DI\autowire()->constructorParameter('type', Stage\Primary::class), Span\SpanInterface::class => DI\factory([Span\Primary::class, 'create']), - Span\Factory::class => DI\object()->constructorParameter('type', Span\Primary::class), - TransportFactory::class => DI\object()->constructorParameter('type', TransportPrimary::class), - DebugMode::class => DI\object()->constructorParameter('debugMode', $this->isDebug), - RemoteConnectionInterface::class => DI\object( + Span\Factory::class => DI\autowire()->constructorParameter('type', Span\Primary::class), + TransportFactory::class => DI\autowire()->constructorParameter('type', TransportPrimary::class), + DebugMode::class => DI\create()->constructor($this->isDebug), + RemoteConnectionInterface::class => DI\autowire( $this->isDebug ? DebugExternalConnection::class : Connection::class ), - LocalConnectionInterface::class => DI\object( + LocalConnectionInterface::class => DI\autowire( \Driver\System\LocalConnectionLoader::class ) ]; - - return $output; } - public function getForTests() + /** + * @return + */ + public function getForTests(): array { return array_merge( $this->get(), From a29d4cafa5f4e819ae95cba7287cf8b9a7b1cd80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Thu, 23 Jun 2022 11:43:10 +0200 Subject: [PATCH 02/29] #21 - Config loader changes, refactoring and cleanup --- config/{test.yaml => config.yaml} | 0 src/System/Configuration.php | 126 +++++++-------- src/System/Configuration/FileCollector.php | 71 +++++++++ src/System/Configuration/FileLoader.php | 28 ++++ src/System/Configuration/FolderCollection.php | 46 ++++++ .../Configuration/FolderCollectionFactory.php | 71 +++++++++ src/System/Configuration/SearchPath.php | 116 -------------- src/System/Configuration/YamlLoader.php | 126 --------------- src/System/DependencyConfig.php | 2 +- src/Tests/Unit/Pipeline/Span/PrimaryTest.php | 9 +- src/Tests/Unit/Pipeline/Stage/PrimaryTest.php | 9 +- .../System/Configuration/YamlLoaderTest.php | 59 ------- src/Tests/Unit/System/ConfigurationTest.php | 144 ++---------------- 13 files changed, 294 insertions(+), 513 deletions(-) rename config/{test.yaml => config.yaml} (100%) create mode 100644 src/System/Configuration/FileCollector.php create mode 100755 src/System/Configuration/FileLoader.php create mode 100755 src/System/Configuration/FolderCollection.php create mode 100644 src/System/Configuration/FolderCollectionFactory.php delete mode 100755 src/System/Configuration/SearchPath.php delete mode 100755 src/System/Configuration/YamlLoader.php delete mode 100755 src/Tests/Unit/System/Configuration/YamlLoaderTest.php diff --git a/config/test.yaml b/config/config.yaml similarity index 100% rename from config/test.yaml rename to config/config.yaml diff --git a/src/System/Configuration.php b/src/System/Configuration.php index 37d3548..39bb65b 100755 --- a/src/System/Configuration.php +++ b/src/System/Configuration.php @@ -1,63 +1,58 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/8/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\System; -use Driver\System\Configuration\YamlLoader; + +use Driver\System\Configuration\FileCollector; +use Driver\System\Configuration\FileLoader; use Symfony\Component\Yaml\Exception\ParseException; use Symfony\Component\Yaml\Yaml; +use function array_keys; +use function array_merge; +use function array_reduce; +use function array_walk; +use function count; +use function explode; +use function is_array; +use function is_int; +use function is_string; + class Configuration { - /** @var YamlLoader $loader */ - protected $loader; - - protected $nodes = [ - 'pipelines' => [] - ]; + private FileCollector $fileCollector; + private FileLoader $loader; + private array $nodes = ['pipelines' => []]; + private array $files = []; - protected $files = []; - - public function __construct(YamlLoader $loader) + public function __construct(FileCollector $fileCollector, FileLoader $fileLoader) { - $this->loader = $loader; + $this->fileCollector = $fileCollector; + $this->loader = $fileLoader; } - public function getNodes() + public function getNodes(): array { if (!count($this->files)) { - $this->loadAllConfiguration(); + foreach ($this->fileCollector->get() as $file) { + $this->loadConfigurationFor($file); + }; } return $this->nodes; } + /** + * @return mixed + */ public function getNode($node) { $path = explode('/', $node); $nodes = $this->getNodes(); return array_reduce($path, function($nodes, $item) { - if (isset($nodes[$item])) { - return $nodes[$item]; - } else { - return null; - } + return $nodes[$item] ?? null; }, $nodes); } @@ -68,7 +63,7 @@ public function getNodeString($node): string return is_string($value) ? $value : ''; } - protected function loadConfigurationFor($file) + private function loadConfigurationFor($file): void { if (!isset($this->files[$file])) { try { @@ -89,11 +84,9 @@ protected function loadConfigurationFor($file) throw $e; } } - - return $this->files[$file]; } - private function recursiveMerge(array $array1, array $array2) + private function recursiveMerge(array $array1, array $array2): array { $merged = $array1; @@ -111,10 +104,8 @@ private function recursiveMerge(array $array1, array $array2) /** * Special handling for pipelines as they don't exactly follow the key/value pattern. - * - * @param $input */ - protected function mergePipelines($new) + private function mergePipelines(array $new): array { if (!isset($this->nodes['pipelines']) || !count($this->nodes['pipelines'])) { return $new; @@ -146,7 +137,7 @@ protected function mergePipelines($new) }, []); } - protected function mergePipeline($existing, $new) + private function mergePipeline($existing, $new) { array_walk($new, function($value) use (&$existing) { $existing = $this->mergeStageIntoPipeline($existing, $value); @@ -155,24 +146,30 @@ protected function mergePipeline($existing, $new) return $existing; } - protected function mergeStageIntoPipeline($existing, $newStage) + private function mergeStageIntoPipeline(array $existing, array $newStage): array { $matched = false; - $output = array_reduce(array_keys($existing), function($carry, $existingKey) use ($existing, $newStage, &$matched) { - $existingStage = $existing[$existingKey]; - $currentMatch = isset($newStage['name']) && isset($existingStage['name']) && $newStage['name'] == $existingStage['name']; - $matched = $matched || $currentMatch; - - if (!$currentMatch) { - return $carry; - } + $output = array_reduce( + array_keys($existing), + function($carry, $existingKey) use ($existing, $newStage, &$matched) { + $existingStage = $existing[$existingKey]; + $currentMatch = isset($newStage['name']) + && isset($existingStage['name']) + && $newStage['name'] == $existingStage['name']; + $matched = $matched || $currentMatch; + + if (!$currentMatch) { + return $carry; + } - $carry[$existingKey] = array_merge($existingStage, $newStage); - $carry[$existingKey]['actions'] = array_merge($existingStage['actions'], $newStage['actions']); + $carry[$existingKey] = array_merge($existingStage, $newStage); + $carry[$existingKey]['actions'] = array_merge($existingStage['actions'], $newStage['actions']); - return $carry; - }, $existing); + return $carry; + }, + $existing + ); if (!$matched) { $output[] = $newStage; @@ -180,21 +177,4 @@ protected function mergeStageIntoPipeline($existing, $newStage) return $output; } - - protected function consolidateStage($existing, $new) - { - - } - - protected function stripFileExtension($file) - { - return pathinfo($file, PATHINFO_FILENAME); - } - - protected function loadAllConfiguration() - { - foreach ($this->loader->get() as $file) { - $this->loadConfigurationFor((string)$file); - }; - } } diff --git a/src/System/Configuration/FileCollector.php b/src/System/Configuration/FileCollector.php new file mode 100644 index 0000000..d9cc3c0 --- /dev/null +++ b/src/System/Configuration/FileCollector.php @@ -0,0 +1,71 @@ +create(self::ALLOWED_FOLDERS); + $output = []; + + foreach ($folderCollection as $folder) { + $files = array_filter(self::ALLOWED_FILES, function ($file) use ($folder) { + return file_exists($folder . '/' . $file . self::FILE_EXTENSION); + }); + + $output = array_merge($output, array_map(function ($file) use ($folder) { + return $folder . '/' . $file . self::FILE_EXTENSION; + }, $files)); + } + + return array_unique(array_reverse($output)); + } + + /** + * @return string[] + */ + public function getIndividual(string $file): array + { + $folderCollection = (new FolderCollectionFactory())->create(self::ALLOWED_FOLDERS); + $output = []; + + foreach ($folderCollection as $folder) { + $path = $folder . '/' . $file . self::FILE_EXTENSION; + + if (file_exists($path)) { + $output[] = $path; + } + } + + return $output; + } +} diff --git a/src/System/Configuration/FileLoader.php b/src/System/Configuration/FileLoader.php new file mode 100755 index 0000000..870e2d9 --- /dev/null +++ b/src/System/Configuration/FileLoader.php @@ -0,0 +1,28 @@ +folders = $folders; + } + + public function current(): string + { + return $this->folders[$this->position]; + } + + public function next(): void + { + ++$this->position; + } + + public function key(): int + { + return $this->position; + } + + public function valid(): bool + { + return isset($this->folders[$this->position]); + } + + public function rewind(): void + { + $this->position = 0; + } +} diff --git a/src/System/Configuration/FolderCollectionFactory.php b/src/System/Configuration/FolderCollectionFactory.php new file mode 100644 index 0000000..1748397 --- /dev/null +++ b/src/System/Configuration/FolderCollectionFactory.php @@ -0,0 +1,71 @@ +getFolders($this->getSearchPaths(), $allowedFolders)); + } + + /** + * @return string[] + */ + private function getSearchPaths(): array + { + $directory = realpath($_SERVER['SCRIPT_FILENAME']); + if (strpos($directory, self::VENDOR_DIRECTORY) !== false) { + list($rootDir) = explode(self::VENDOR_DIRECTORY, $directory); + return array_merge([$rootDir], $this->getVendorDirectories($rootDir)); + } + return [dirname($directory, 2)]; + } + + /** + * @return string[] + */ + private function getVendorDirectories(string $path): array + { + return glob($path . self::VENDOR_DIRECTORY . "/*/*/", GLOB_ONLYDIR); + } + + /** + * @param string[] $paths + * @param string[] $allowedFolders + * @return string[] + */ + private function getFolders(array $paths, array $allowedFolders): array + { + return array_reduce($paths, function ($acc, $path) use ($allowedFolders) { + $path = rtrim($path, '/') . '/'; + + $folders = array_filter($allowedFolders, function ($folder) use ($path) { + return file_exists($path . $folder); + }); + + return array_merge($acc, array_map(function ($folder) use ($path) { + return $path . $folder; + }, $folders)); + }, []); + } +} diff --git a/src/System/Configuration/SearchPath.php b/src/System/Configuration/SearchPath.php deleted file mode 100755 index 514d6bd..0000000 --- a/src/System/Configuration/SearchPath.php +++ /dev/null @@ -1,116 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/6/16 - * @package default - **/ - -namespace Driver\System\Configuration; - -class SearchPath implements \Iterator -{ - const VENDOR_DIRECTORY = 'vendor'; - - private $searchPaths; - private $allowedFolders; - private $position = 0; - private $folders; - - public function __construct($directory = __DIR__, $allowedFolders) - { - $this->allowedFolders = $allowedFolders; - $this->searchPaths = $this->format($directory); - $this->folders = $this->findFolders($this->searchPaths); - } - - private function format($directory) - { - $hasVendorDir = false; - if (strpos($directory, self::VENDOR_DIRECTORY) !== false) { - list($initial, $continue) = explode(self::VENDOR_DIRECTORY, $directory); - $initial = [ $initial ]; - $continue = self::VENDOR_DIRECTORY . $continue; - $hasVendorDir = true; - } else { - $topOfSearch = strlen(realpath($directory.'/../../../')); - $initial = [ substr($directory, 0, $topOfSearch) ]; - $continue = substr($directory, $topOfSearch+1); - } - - $paths = $this->merge(array_merge($initial, explode('/', $continue))); - if ($hasVendorDir) { - $paths = array_merge($paths, $this->loadVendorDirectories($initial[0])); - } - - return $paths; - } - - private function merge($directories) - { - $currentPath = ''; - - return array_reduce($directories, function($acc, $path) use (&$currentPath) { - $currentPath .= $path; - if (substr($currentPath, -1) !== '/') { - $currentPath .= '/'; - } - $acc[] = $currentPath; - return $acc; - }, []); - } - - private function loadVendorDirectories($path) - { - return glob($path . self::VENDOR_DIRECTORY . "/*/*/", GLOB_ONLYDIR); - } - - private function findFolders($paths) - { - return array_reduce($paths, function($acc, $path) { - $folders = array_filter($this->allowedFolders, function($folder) use ($path) { - return file_exists($path . $folder); - }); - - return array_merge($acc, array_map(function($folder) use ($path) { - return $path . $folder; - }, $folders)); - }, []); - } - - public function current() - { - return $this->folders[$this->position]; - } - - public function next() - { - ++$this->position; - } - - public function key() - { - return $this->position; - } - - public function valid() - { - return isset($this->folders[$this->position]); - } - - public function rewind() - { - $this->position = 0; - } -} \ No newline at end of file diff --git a/src/System/Configuration/YamlLoader.php b/src/System/Configuration/YamlLoader.php deleted file mode 100755 index bd9a44c..0000000 --- a/src/System/Configuration/YamlLoader.php +++ /dev/null @@ -1,126 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/15/16 - * @package default - **/ - -namespace Driver\System\Configuration; - -class YamlLoader -{ - protected $fileExtension = '.yaml'; - - protected $allowedFolders = [ - 'config', - 'config.d' - ]; - - protected $allowedFiles = [ - 'anonymize', - 'pipelines', - 'commands', - 'engines', - 'connections', - 'config', - 'reduce', - 'environments' - ]; - - public function get() - { - return $this->getAllFiltered(); - } - - public function getIndividual($file) - { - return $this->getFiltered($file); - } - - protected function getFiltered($file) - { - $searchPath = new SearchPath(__DIR__, $this->allowedFolders); - $output = []; - - foreach ($searchPath as $folder) { - $path = $folder . '/' . $file . $this->fileExtension; - - if (file_exists($path)) { - $output[] = $path; - } - } - - return $output; - } - - protected function getAllFiltered() - { - $searchPath = new SearchPath(__DIR__, $this->allowedFolders); - $output = []; - - foreach ($searchPath as $folder) { - $files = array_filter($this->allowedFiles, function($file) use ($folder) { - return file_exists($folder . '/' . $file . $this->fileExtension); - }); - - $output = array_merge($output, array_map(function($file) use ($folder) { - return $folder . '/' . $file . $this->fileExtension; - }, $files)); - } - - return array_unique(array_reverse($output)); - } - - /** - * Returns the contents of a yaml file. - * - * @param $file - * @return string - * @throws \Exception - */ - public function load($file) - { - if (!$file || !file_exists($file)) { - throw new \Exception("Filename: {$file} doesn't exist."); - } - return file_get_contents($file); - } - - protected function isAllowedFile($path, $additionalFiles = []) - { - if (!is_array($additionalFiles) && !$additionalFiles) { - $additionalFiles = []; - } else if (!is_array($additionalFiles) && $additionalFiles) { - $additionalFiles = [$additionalFiles]; - } - - if (count($additionalFiles)) { - $fileSearch = $additionalFiles; - } else { - $fileSearch = $this->allowedFiles; - } - - $usedAllowedFolders = array_filter($this->allowedFolders, function($allowedFolder) use ($path) { - return strpos($path, '/'.$allowedFolder.'/') !== false; - }); - - $usedAllowedFiles = array_filter($fileSearch, function($allowedFile) use ($path) { - $filename = explode('.', $allowedFile)[0] . $this->fileExtension; - return substr($path, 0-strlen($filename)) === $filename; - }); - - return count($usedAllowedFolders) && count($usedAllowedFiles); - } -} \ No newline at end of file diff --git a/src/System/DependencyConfig.php b/src/System/DependencyConfig.php index 965e646..5677c17 100644 --- a/src/System/DependencyConfig.php +++ b/src/System/DependencyConfig.php @@ -40,7 +40,7 @@ class DependencyConfig { private bool $isDebug = false; - public function __construct(bool $isDebug) + public function __construct(bool $isDebug = false) { $this->isDebug = $isDebug; } diff --git a/src/Tests/Unit/Pipeline/Span/PrimaryTest.php b/src/Tests/Unit/Pipeline/Span/PrimaryTest.php index aaad444..29f0fe9 100644 --- a/src/Tests/Unit/Pipeline/Span/PrimaryTest.php +++ b/src/Tests/Unit/Pipeline/Span/PrimaryTest.php @@ -31,9 +31,12 @@ class PrimaryTest extends \PHPUnit_Framework_TestCase public function testInvokeReturnsTransport() { $pipelineName = Master::DEFAULT_NODE; - $configuration = new Configuration(new Configuration\YamlLoader()); + $configuration = new Configuration(new Configuration\FileCollector(), new Configuration\FileLoader()); $set = DI::getContainer()->make(Primary::class, ['list' => $configuration->getNode('pipelines/' . $pipelineName)]); - $this->assertTrue(is_a($set(new Transport($pipelineName, [], [], new \Driver\System\Logs\Primary()), true), TransportInterface::class)); + $this->assertTrue(is_a( + $set(new Transport($pipelineName, [], [], null, new \Driver\System\Logs\Primary()), true), + TransportInterface::class + )); } -} \ No newline at end of file +} diff --git a/src/Tests/Unit/Pipeline/Stage/PrimaryTest.php b/src/Tests/Unit/Pipeline/Stage/PrimaryTest.php index a09312a..1d90be8 100644 --- a/src/Tests/Unit/Pipeline/Stage/PrimaryTest.php +++ b/src/Tests/Unit/Pipeline/Stage/PrimaryTest.php @@ -31,9 +31,12 @@ class PrimaryTest extends \PHPUnit_Framework_TestCase public function testInvokeReturnsTransport() { $pipelineName = Master::DEFAULT_NODE; - $configuration = new Configuration(new Configuration\YamlLoader()); + $configuration = new Configuration(new Configuration\FileCollector(), new Configuration\FileLoader()); $set = DI::getContainer()->make(Primary::class, ['actions' => $configuration->getNode('pipelines/' . $pipelineName)]); - $this->assertTrue(is_a($set(new Transport($pipelineName, [], [], new \Driver\System\Logs\Primary()), true), TransportInterface::class)); + $this->assertTrue(is_a( + $set(new Transport($pipelineName, [], [], null, new \Driver\System\Logs\Primary()), true), + TransportInterface::class + )); } -} \ No newline at end of file +} diff --git a/src/Tests/Unit/System/Configuration/YamlLoaderTest.php b/src/Tests/Unit/System/Configuration/YamlLoaderTest.php deleted file mode 100755 index abc9103..0000000 --- a/src/Tests/Unit/System/Configuration/YamlLoaderTest.php +++ /dev/null @@ -1,59 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/15/16 - * @package default - **/ - -namespace Driver\Tests\Unit\System\Configuration; - -use Driver\System\Configuration\YamlLoader; - -class YamlLoaderTest extends \PHPUnit_Framework_TestCase -{ - public function testAllowedFiles() - { - $configuration = new YamlLoader(); - - $method = new \ReflectionMethod($configuration, 'isAllowedFile'); - $method->setAccessible(true); - - $this->assertTrue($method->invoke($configuration, '/var/www/config/pipelines.yaml')); - $this->assertTrue($method->invoke($configuration, '/var/www/config.d/commands.yaml')); - $this->assertTrue($method->invoke($configuration, '/var/www/config.d/connections.yaml')); - - $this->assertFalse($method->invoke($configuration, '/var/www/configuration/chain.yaml')); - $this->assertFalse($method->invoke($configuration, '/var/www/test/chain.yaml')); - $this->assertFalse($method->invoke($configuration, '/var/www/config/input.yaml')); - $this->assertFalse($method->invoke($configuration, '/var/www/config.d/bad-file.yaml')); - } - - public function testGetYamlFilesReturnsArray() - { - $configuration = new YamlLoader(); - - $method = new \ReflectionMethod($configuration, 'get'); - $method->setAccessible(true); - - $count = 0; - $files = []; - foreach ($method->invoke($configuration) as $file) { - $count++; - $files[] = $file; - } - - $this->assertGreaterThanOrEqual(1, $count); - } -} \ No newline at end of file diff --git a/src/Tests/Unit/System/ConfigurationTest.php b/src/Tests/Unit/System/ConfigurationTest.php index fbf36d9..364ba60 100644 --- a/src/Tests/Unit/System/ConfigurationTest.php +++ b/src/Tests/Unit/System/ConfigurationTest.php @@ -1,149 +1,29 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/8/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Tests\Unit\System; -use DI\ContainerBuilder; use Driver\System\Configuration; -use Driver\System\Configuration\YamlFilter; -use Symfony\Component\Yaml\Yaml; class ConfigurationTest extends \PHPUnit_Framework_TestCase { - protected $testFile; - - public function setUp() - { - $this->testFile = realpath(__DIR__ . '../../../config/') . 'test.yaml'; - if (!file_exists($this->testFile)) { - file_put_contents($this->testFile, "test:\n value: unknown"); - } - - parent::setUp(); - } - - public function tearDown() - { - unlink($this->testFile); - - parent::tearDown(); - } - - protected function getConfiguration() - { - $configuration = new Configuration(new Configuration\YamlLoader()); - return $configuration; - } + private Configuration $configuration; - protected function getConfigurationLoadedWithTestFile() + public function setUp(): void { - $configuration = $this->getConfiguration(); - $yamlLoader = new Configuration\YamlLoader(); - - $filePath = $yamlLoader->getIndividual('test'); - - $loaderMethod = new \ReflectionMethod($configuration, 'loadConfigurationFor'); - $loaderMethod->setAccessible(true); - $output = $loaderMethod->invoke($configuration, reset($filePath)); - - return [ - 'output' => $output, - 'configuration' => $configuration - ]; + $this->configuration = new Configuration(new Configuration\FileCollector(), new Configuration\FileLoader()); } - public function testGetAllNodesReturnsInformation() + public function testGetAllNodesReturnsInformation(): void { - $values = $this->getConfigurationLoadedWithTestFile(); - $configuration = $values['configuration']; - - $this->assertInternalType('array', $configuration->getNodes()); + $result = $this->configuration->getNodes(); + $this->assertInternalType('array', $result); + $this->assertNotEmpty($result); } - public function testGetNodeReturnsInformation() + public function testGetNodeReturnsInformation(): void { - $values = $this->getConfigurationLoadedWithTestFile(); - $configuration = $values['configuration']; - - $this->assertSame('unknown', $configuration->getNode('test/value')); - } - - public function testGetAllConfigurationLoadsEverything() - { - $configuration = $this->getConfiguration(); - - $filesProperty = new \ReflectionProperty($configuration, 'files'); - $filesProperty->setAccessible(true); - - $loaderMethod = new \ReflectionMethod($configuration, 'loadAllConfiguration'); - $loaderMethod->setAccessible(true); - $loaderMethod->invoke($configuration); - - $this->assertTrue(count($filesProperty->getValue($configuration)) > 0); - } - - public function testRemovesFileSuffix() - { - $configuration = $this->getConfiguration(); - - $stripMethod = new \ReflectionMethod($configuration, 'stripFileExtension'); - $stripMethod->setAccessible(true); - - $this->assertEquals('test', $stripMethod->invoke($configuration, 'test.yaml')); - } - - public function testGetConfigurationFileTestLoadsArrayValues() - { - $values = $this->getConfigurationLoadedWithTestFile(); - $configuration = $values['configuration']; - $output = $values['output']; - - $this->assertEquals('unknown', $output['test']['value']); - $this->assertTrue(count($configuration) > 0); - } - - public function testGetConfigurationFileLoadsClassArrays() - { - $values = $this->getConfigurationLoadedWithTestFile(); - $configuration = $values['configuration']; - - $fileArray = new \ReflectionProperty($configuration, 'files'); - $fileArray->setAccessible(true); - $nodeArray = new \ReflectionProperty($configuration, 'nodes'); - $nodeArray->setAccessible(true); - $nodes = $nodeArray->getValue($configuration); - - $this->assertSame(1, count($fileArray->getValue($configuration))); - $this->assertSame('unknown', $nodes['test']['value']); - } - - public function testGetConfigurationFileLoadsData() - { - $yamlLoader = new Configuration\YamlLoader(); - $configuration = $this->getConfiguration(); - - $files = $yamlLoader->get(); - - $loaderMethod = new \ReflectionMethod($configuration, 'loadConfigurationFor'); - $loaderMethod->setAccessible(true); - $configuration = $loaderMethod->invoke($configuration, reset($files)); - - $this->assertTrue(count($configuration) > 0); + $this->assertSame('unknown', $this->configuration->getNode('test/value')); } -} \ No newline at end of file +} From 053c764cf34dd388ccb77a1b71ed4ff96b34ad87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Mon, 27 Jun 2022 11:20:02 +0200 Subject: [PATCH 03/29] #22 - switching native PDO to custom PersistentPDO which prevents timeouts --- composer.json | 1 + src/Engines/ConnectionInterface.php | 2 +- src/Engines/ConnectionTrait.php | 34 ++++------- src/Engines/MySql/Transformation.php | 3 +- src/Engines/PersistentPDO.php | 87 ++++++++++++++++++++++++++++ src/System/LocalConnectionLoader.php | 3 +- 6 files changed, 103 insertions(+), 27 deletions(-) create mode 100644 src/Engines/PersistentPDO.php diff --git a/composer.json b/composer.json index 995864c..19b2bb2 100755 --- a/composer.json +++ b/composer.json @@ -3,6 +3,7 @@ "description": "A database production to sandbox utility to sanitize data.", "type": "library", "require": { + "ext-pdo": "*", "php-di/php-di": "^6.0", "aws/aws-sdk-php": "^3.19", "symfony/yaml": ">=2.3", diff --git a/src/Engines/ConnectionInterface.php b/src/Engines/ConnectionInterface.php index 02f07a8..51ff076 100644 --- a/src/Engines/ConnectionInterface.php +++ b/src/Engines/ConnectionInterface.php @@ -23,7 +23,7 @@ interface ConnectionInterface { public function isAvailable(): bool; - public function getConnection(): \PDO; + public function getConnection(): PersistentPDO; public function getDSN(): string; diff --git a/src/Engines/ConnectionTrait.php b/src/Engines/ConnectionTrait.php index 5eec3d8..7d0c860 100644 --- a/src/Engines/ConnectionTrait.php +++ b/src/Engines/ConnectionTrait.php @@ -1,41 +1,27 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/3/16 - * @package default - **/ +declare(strict_types=1); namespace Driver\Engines; +use PDO; + trait ConnectionTrait { - private $connection; + private ?PersistentPDO $connection = null; - public function getConnection(): \PDO + public function getConnection(): PersistentPDO { if (!$this->connection) { $options = [ - \PDO::ATTR_ERRMODE => \PDO::ERRMODE_EXCEPTION, - \PDO::ATTR_DEFAULT_FETCH_MODE => \PDO::FETCH_ASSOC, - \PDO::ATTR_EMULATE_PREPARES => false + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_EMULATE_PREPARES => false ]; - $this->connection = new \PDO($this->getDSN(), $this->getUser(), $this->getPassword(), $options); + $this->connection = new PersistentPDO($this->getDSN(), $this->getUser(), $this->getPassword(), $options); } return $this->connection; } -} \ No newline at end of file +} diff --git a/src/Engines/MySql/Transformation.php b/src/Engines/MySql/Transformation.php index 2003b73..1c344de 100644 --- a/src/Engines/MySql/Transformation.php +++ b/src/Engines/MySql/Transformation.php @@ -21,6 +21,7 @@ use Driver\Commands\CommandInterface; use Driver\Engines\MySql\Sandbox\Utilities; +use Driver\Engines\PersistentPDO; use Driver\Engines\RemoteConnectionInterface; use Driver\Pipeline\Environment\EnvironmentInterface; use Driver\Pipeline\Transport\Status; @@ -71,7 +72,7 @@ public function getProperties() return $this->properties; } - private function applyTransformationsTo(\PDO $connection, $transformations) + private function applyTransformationsTo(PersistentPDO $connection, $transformations) { array_walk($transformations, function ($query) use ($connection) { try { diff --git a/src/Engines/PersistentPDO.php b/src/Engines/PersistentPDO.php new file mode 100644 index 0000000..875e293 --- /dev/null +++ b/src/Engines/PersistentPDO.php @@ -0,0 +1,87 @@ +dsn = $dsn; + $this->username = $username; + $this->password = $password; + $this->options = $options; + $this->pdo = $this->createPDO(); + } + + /** + * @throws PDOException + */ + public function __call(string $name, array $arguments) + { + try { + $this->pdo->query('SELECT 1')->fetchColumn(); + } catch (PDOException $e) { + if ($e->errorInfo[0] !== self::MYSQL_GENERAL_ERROR_CODE + || $e->errorInfo[1] !== self::SERVER_HAS_GONE_AWAY_ERROR_CODE + ) { + throw $e; + } + $this->pdo = $this->createPDO(); + } + return $this->pdo->$name(...$arguments); + } + + /** + * @throws PDOException + */ + private function createPDO(): PDO + { + $pdo = new PDO($this->dsn, $this->username, $this->password, $this->options); + $pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); + return $pdo; + } +} diff --git a/src/System/LocalConnectionLoader.php b/src/System/LocalConnectionLoader.php index 596c474..0bad9b4 100644 --- a/src/System/LocalConnectionLoader.php +++ b/src/System/LocalConnectionLoader.php @@ -10,6 +10,7 @@ use DI\Container; use Driver\Engines\ConnectionInterface; use Driver\Engines\LocalConnectionInterface; +use Driver\Engines\PersistentPDO; class LocalConnectionLoader implements LocalConnectionInterface { @@ -35,7 +36,7 @@ public function __construct( $this->container = $container; } - public function getConnection(): \PDO + public function getConnection(): PersistentPDO { return $this->get()->getConnection(); } From 103d2c1a2c291f7f12244b3bf933178a80c0daa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Mon, 27 Jun 2022 12:37:37 +0200 Subject: [PATCH 04/29] #23 - fix for Access Denied issue when importing database from S3 --- src/Engines/MySql/Sandbox/Export.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Engines/MySql/Sandbox/Export.php b/src/Engines/MySql/Sandbox/Export.php index 8e347d7..bb49e10 100755 --- a/src/Engines/MySql/Sandbox/Export.php +++ b/src/Engines/MySql/Sandbox/Export.php @@ -116,6 +116,7 @@ private function assembleCommand($environmentName, $ignoredTables) ], $this->getIgnoredTables($ignoredTables))); $command .= " {$this->connection->getDatabase()} "; + $command .= "| sed -e 's/DEFINER[ ]*=[ ]*[^*]*\*/\*/' "; if ($this->compressOutput()) { $command .= ' ' . implode(' ', [ From b76861979fcf849a2f222135432746f25709f68d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Mon, 27 Jun 2022 17:59:44 +0200 Subject: [PATCH 05/29] #24 - Smarter anonymization - work in progress --- config/commands.yaml | 2 + config/pipelines.yaml | 54 ++++--- .../MySql/Transformation/Anonymize.php | 18 +-- .../MySql/Transformation/Anonymize/Seed.php | 18 +++ .../MySql/Transformation/UpdateValues.php | 143 ++++++++++++++++++ .../Transformation/UpdateValues/Join.php | 34 +++++ .../UpdateValues/QueryBuilder.php | 40 +++++ .../Transformation/UpdateValues/Value.php | 27 ++++ 8 files changed, 301 insertions(+), 35 deletions(-) create mode 100644 src/Engines/MySql/Transformation/UpdateValues.php create mode 100644 src/Engines/MySql/Transformation/UpdateValues/Join.php create mode 100644 src/Engines/MySql/Transformation/UpdateValues/QueryBuilder.php create mode 100644 src/Engines/MySql/Transformation/UpdateValues/Value.php diff --git a/config/commands.yaml b/config/commands.yaml index a589bf2..39668a6 100644 --- a/config/commands.yaml +++ b/config/commands.yaml @@ -15,6 +15,8 @@ commands: class: \Driver\Engines\MySql\Transformation\Anonymize reduce: class: \Driver\Engines\MySql\Transformation\Reduce + update-values: + class: \Driver\Engines\MySql\Transformation\UpdateValues export-data-from-sandbox: class: \Driver\Engines\MySql\Sandbox\Export upload-data-to-s3: diff --git a/config/pipelines.yaml b/config/pipelines.yaml index b6f9515..786916b 100755 --- a/config/pipelines.yaml +++ b/config/pipelines.yaml @@ -33,6 +33,8 @@ pipelines: sort: 100 - name: anonymize sort: 200 + - name: update-values + sort: 300 - name: repeat-commands sort: 400 @@ -56,46 +58,48 @@ pipelines: - name: setup # pipeline stage sort: 100 actions: - - name: connect - sort: 100 - - name: check-filesystem - sort: 200 - - name: start-sandbox - sort: 300 + - name: connect + sort: 100 + - name: check-filesystem + sort: 200 + - name: start-sandbox + sort: 300 - name: import sort: 200 actions: - - name: export-data-from-system-primary - sort: 100 - - name: import-data-into-sandbox - sort: 200 + - name: export-data-from-system-primary + sort: 100 + - name: import-data-into-sandbox + sort: 200 - name: global-commands sort: 300 actions: - - name: reduce - sort: 100 - - name: anonymize - sort: 200 + - name: reduce + sort: 100 + - name: anonymize + sort: 200 + - name: update-values + sort: 300 - name: repeat-commands sort: 400 actions: - - name: run-transformations - sort: 100 - - name: connect - sort: 200 - - name: export-data-from-sandbox - sort: 300 - - name: upload-data-to-s3 - sort: 400 + - name: run-transformations + sort: 100 + - name: connect + sort: 200 + - name: export-data-from-sandbox + sort: 300 + - name: upload-data-to-s3 + sort: 400 - name: shutdown sort: 500 actions: - - name: shutdown-sandbox - sort: 100 + - name: shutdown-sandbox + sort: 100 import-s3: - name: export-s3-db-on-local @@ -110,4 +114,4 @@ pipelines: - name: import-data-from-system-primary sort: 100 - empty: [] + empty: [ ] diff --git a/src/Engines/MySql/Transformation/Anonymize.php b/src/Engines/MySql/Transformation/Anonymize.php index 841e799..afdba44 100644 --- a/src/Engines/MySql/Transformation/Anonymize.php +++ b/src/Engines/MySql/Transformation/Anonymize.php @@ -121,16 +121,12 @@ private function anonymize(string $table, array $columns) /** * Many thanks to this method for inspiration: - * https://github.com/DivanteLtd/anonymizer/blob/master/lib/anonymizer/model/database/column.rb+ - * - * @param string $table - * @param string $columnName - * @param $description - * @return string + * https://github.com/DivanteLtd/anonymizer/blob/master/lib/anonymizer/model/database/column.rb */ - private function queryEmail($type, $columnName): string + private function queryEmail(string $type, string $columnName): string { - return "CONCAT((SELECT CONCAT(MD5(FLOOR((NOW() + RAND()) * (RAND() * RAND() / RAND()) + RAND())))), \"@\", SUBSTRING({$columnName}, LOCATE('@', {$columnName}) + 1))"; + $salt = $this->seed->getSalt(); + return "CONCAT(SELECT MD5(CONCAT(\"${$salt}\", ${columnName})), \"@\", SUBSTRING({$columnName}, LOCATE('@', {$columnName}) + 1))"; } private function queryFullName(): string @@ -140,14 +136,16 @@ private function queryFullName(): string return "(SELECT CONCAT_WS(' ', ${table}.firstname, ${table}.lastname) FROM ${table} ORDER BY RAND() LIMIT 1)"; } - private function queryGeneral($type): string + private function queryGeneral($type, $columnName): string { if ($type === "general") { return "(SELECT MD5(FLOOR((NOW() + RAND()) * (RAND() * RAND() / RAND()) + RAND())))"; } $table = Seed::FAKE_USER_TABLE; - return "(SELECT ${table}.${type} FROM ${table} ORDER BY RAND() LIMIT 1)"; + $salt = $this->seed->getSalt(); + $count = $this->seed->getCount(); + return "(SELECT ${table}.${type} FROM ${table} WHERE id = (SELECT 1 + MOD(ORD(MD5(CONCAT(\"${salt}\", ${columnName}))), ${count})))"; } private function queryAddress(): string diff --git a/src/Engines/MySql/Transformation/Anonymize/Seed.php b/src/Engines/MySql/Transformation/Anonymize/Seed.php index 5da161f..386d8b5 100644 --- a/src/Engines/MySql/Transformation/Anonymize/Seed.php +++ b/src/Engines/MySql/Transformation/Anonymize/Seed.php @@ -10,17 +10,22 @@ use Driver\Engines\RemoteConnectionInterface; use Driver\System\Configuration; +use function uniqid; + class Seed { const FAKE_USER_TABLE = 'fake_users'; private RemoteConnectionInterface $connection; private Configuration $configuration; + private string $salt; + private int $count = 0; public function __construct(Configuration $configuration, RemoteConnectionInterface $connection) { $this->connection = $connection; $this->configuration = $configuration; + $this->salt = uniqid(); } public function initialize(): void @@ -41,6 +46,7 @@ public function initialize(): void $params = array_combine($bind, array_values($seed)); $this->connection->getConnection()->prepare($query)->execute($params); + $this->count++; } } @@ -49,9 +55,20 @@ public function destroy() $this->clean(); } + public function getSalt(): string + { + return $this->salt; + } + + public function getCount(): int + { + return $this->count; + } + private function clean() { $this->connection->getConnection()->query('DROP TABLE IF EXISTS ' . self::FAKE_USER_TABLE); + $this->count = 0; } private function createTable() @@ -60,6 +77,7 @@ private function createTable() $this->connection->getConnection()->query(<<configuration = $configuration; + $this->connection = $connection; + $this->logger = $logger; + $this->output = $output; + $this->queryBuilder = $queryBuilder; + $this->properties = $properties; + parent::__construct('mysql-transformation-update-values'); + } + + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface + { + $config = $this->configuration->getNode('update-values'); + if (isset($config['disabled']) && $config['disabled'] === true) { + return $transport->withStatus(new Status('mysql-transformation-update-values', 'success')); + } + + if (!isset($config['tables'])) { + return $transport->withStatus(new Status('mysql-transformation-update-values', 'success')); + } + + $transport->getLogger()->notice("Beginning table updates from update-values.yaml."); + $this->output->writeln("Beginning table updates from update-values.yaml."); + + foreach ($config['tables'] as $table => $data) { + $this->update($table, $data); + } + + $transport->getLogger()->notice("Database has been updated."); + $this->output->writeln("Database has been updated."); + + return $transport->withStatus(new Status('mysql-transformation-update-values', 'success')); + } + + public function getProperties(): array + { + return $this->properties; + } + + private function update(string $table, array $data): void + { + $joins = $this->buildJoins($data); + $values = $this->buildValues($data); + $query = $this->queryBuilder->build($table, $values, $joins); + if (!$query) { + return; + } + $this->connection->getConnection()->query($query); + } + + /** + * @return Join[] + */ + private function buildJoins(array $data): array + { + if (empty($data['joins'])) { + return []; + } + + if (!is_array($data['joins'])) { + throw new InvalidArgumentException('Joins must be an array.'); + } + + $joins = []; + foreach ($data['joins'] as $joinData) { + foreach (['table', 'alias', 'on'] as $key) { + if (empty($joinData['joins'][$key]) || !is_string($joinData['joins'][$key])) { + throw new InvalidArgumentException( + sprintf('Join option "%s" must be a non-empty string.', $key) + ); + } + } + $joins[] = new Join($joinData['table'], $joinData['alias'], $joinData['on']); + } + return $joins; + } + + /** + * @return Value[] + */ + private function buildValues(array $data): array + { + if (empty($data['values'])) { + return []; + } + + if (!is_array($data['values'])) { + throw new InvalidArgumentException('Values must be an array.'); + } + + $values = []; + foreach ($data['values'] as $joinData) { + foreach (['field', 'value'] as $key) { + if (empty($joinData['values'][$key]) || !is_string($joinData['values'][$key])) { + throw new InvalidArgumentException( + sprintf('Value option "%s" must be a non-empty string.', $key) + ); + } + } + $values[] = new Join($joinData['table'], $joinData['alias'], $joinData['on']); + } + return $values; + } +} diff --git a/src/Engines/MySql/Transformation/UpdateValues/Join.php b/src/Engines/MySql/Transformation/UpdateValues/Join.php new file mode 100644 index 0000000..f2779f4 --- /dev/null +++ b/src/Engines/MySql/Transformation/UpdateValues/Join.php @@ -0,0 +1,34 @@ +table = $table; + $this->alias = $alias; + $this->on = $on; + } + + public function getTable(): string + { + return $this->table; + } + + public function getAlias(): string + { + return $this->alias; + } + + public function getOn(): string + { + return $this->on; + } +} diff --git a/src/Engines/MySql/Transformation/UpdateValues/QueryBuilder.php b/src/Engines/MySql/Transformation/UpdateValues/QueryBuilder.php new file mode 100644 index 0000000..c83aeec --- /dev/null +++ b/src/Engines/MySql/Transformation/UpdateValues/QueryBuilder.php @@ -0,0 +1,40 @@ +getField() . ' = ' . $value->getValue(); + } + + $query .= " FROM ${table}"; + + foreach ($joins as $join) { + if (!$join instanceof Join) { + throw new InvalidArgumentException('Join object expected.'); + } + $query .= ' INNER JOIN ' . $join->getTable() . ' ' . $join->getAlias() . ' ON ' . $join->getOn(); + } + + return $query; + } +} diff --git a/src/Engines/MySql/Transformation/UpdateValues/Value.php b/src/Engines/MySql/Transformation/UpdateValues/Value.php new file mode 100644 index 0000000..91b0115 --- /dev/null +++ b/src/Engines/MySql/Transformation/UpdateValues/Value.php @@ -0,0 +1,27 @@ +field = $field; + $this->value = $value; + } + + public function getField(): string + { + return $this->field; + } + + public function getValue(): string + { + return $this->value; + } +} From 22b62e766acf54256bc23e0afe646b67198923f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Wed, 29 Jun 2022 11:09:14 +0200 Subject: [PATCH 06/29] #24 - Smarter anonymization - work in progress --- .../MySql/Transformation/Anonymize.php | 8 ++-- .../MySql/Transformation/UpdateValues.php | 18 ++++++-- .../UpdateValues/QueryBuilder.php | 46 ++++++++++--------- src/System/Configuration/FileCollector.php | 3 +- 4 files changed, 43 insertions(+), 32 deletions(-) diff --git a/src/Engines/MySql/Transformation/Anonymize.php b/src/Engines/MySql/Transformation/Anonymize.php index afdba44..9411866 100644 --- a/src/Engines/MySql/Transformation/Anonymize.php +++ b/src/Engines/MySql/Transformation/Anonymize.php @@ -94,7 +94,7 @@ private function anonymize(string $table, array $columns) foreach ($columns as $columnName => $description) { try { $method = $this->getTypeMethod($description); - $select = $this->$method($description['type'] ?? 'general', $columnName); + $select = $this->$method($description['type'] ?? 'general', $columnName, $table); $query = "UPDATE `${table}` SET `${columnName}` = ${select} WHERE `${table}`.`${columnName}` IS NOT NULL;"; @@ -126,7 +126,7 @@ private function anonymize(string $table, array $columns) private function queryEmail(string $type, string $columnName): string { $salt = $this->seed->getSalt(); - return "CONCAT(SELECT MD5(CONCAT(\"${$salt}\", ${columnName})), \"@\", SUBSTRING({$columnName}, LOCATE('@', {$columnName}) + 1))"; + return "CONCAT(MD5(CONCAT(\"${salt}\", ${columnName})), \"@\", SUBSTRING({$columnName}, LOCATE('@', {$columnName}) + 1))"; } private function queryFullName(): string @@ -136,7 +136,7 @@ private function queryFullName(): string return "(SELECT CONCAT_WS(' ', ${table}.firstname, ${table}.lastname) FROM ${table} ORDER BY RAND() LIMIT 1)"; } - private function queryGeneral($type, $columnName): string + private function queryGeneral(string $type, string $columnName, string $mainTable): string { if ($type === "general") { return "(SELECT MD5(FLOOR((NOW() + RAND()) * (RAND() * RAND() / RAND()) + RAND())))"; @@ -145,7 +145,7 @@ private function queryGeneral($type, $columnName): string $table = Seed::FAKE_USER_TABLE; $salt = $this->seed->getSalt(); $count = $this->seed->getCount(); - return "(SELECT ${table}.${type} FROM ${table} WHERE id = (SELECT 1 + MOD(ORD(MD5(CONCAT(\"${salt}\", ${columnName}))), ${count})))"; + return "(SELECT ${table}.${type} FROM ${table} WHERE ${table}.id = (SELECT 1 + MOD(ORD(MD5(CONCAT(\"${salt}\", ${mainTable}.${columnName}))), ${count})) LIMIT 1)"; } private function queryAddress(): string diff --git a/src/Engines/MySql/Transformation/UpdateValues.php b/src/Engines/MySql/Transformation/UpdateValues.php index e2e7b2e..9d4e163 100644 --- a/src/Engines/MySql/Transformation/UpdateValues.php +++ b/src/Engines/MySql/Transformation/UpdateValues.php @@ -14,6 +14,7 @@ use Driver\Pipeline\Transport\TransportInterface; use Driver\System\Configuration; use Driver\System\Logs\LoggerInterface; +use Exception; use InvalidArgumentException; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Output\ConsoleOutput; @@ -84,7 +85,14 @@ private function update(string $table, array $data): void if (!$query) { return; } - $this->connection->getConnection()->query($query); + $connection = $this->connection->getConnection(); + try { + $connection->beginTransaction(); + $connection->query($query); + $connection->commit(); + } catch (Exception $ex) { + $connection->rollBack(); + } } /** @@ -103,7 +111,7 @@ private function buildJoins(array $data): array $joins = []; foreach ($data['joins'] as $joinData) { foreach (['table', 'alias', 'on'] as $key) { - if (empty($joinData['joins'][$key]) || !is_string($joinData['joins'][$key])) { + if (empty($joinData[$key]) || !is_string($joinData[$key])) { throw new InvalidArgumentException( sprintf('Join option "%s" must be a non-empty string.', $key) ); @@ -128,15 +136,15 @@ private function buildValues(array $data): array } $values = []; - foreach ($data['values'] as $joinData) { + foreach ($data['values'] as $valueData) { foreach (['field', 'value'] as $key) { - if (empty($joinData['values'][$key]) || !is_string($joinData['values'][$key])) { + if (empty($valueData[$key]) || !is_string($valueData[$key])) { throw new InvalidArgumentException( sprintf('Value option "%s" must be a non-empty string.', $key) ); } } - $values[] = new Join($joinData['table'], $joinData['alias'], $joinData['on']); + $values[] = new Value($valueData['field'], $valueData['value']); } return $values; } diff --git a/src/Engines/MySql/Transformation/UpdateValues/QueryBuilder.php b/src/Engines/MySql/Transformation/UpdateValues/QueryBuilder.php index c83aeec..df01a85 100644 --- a/src/Engines/MySql/Transformation/UpdateValues/QueryBuilder.php +++ b/src/Engines/MySql/Transformation/UpdateValues/QueryBuilder.php @@ -6,6 +6,8 @@ use InvalidArgumentException; +use function implode; + class QueryBuilder { /** @@ -14,27 +16,27 @@ class QueryBuilder */ public function build(string $table, array $values = [], array $joins = []): string { - if (empty($values)) { - return ''; - } - - $query = "UPDATE ${$table} SET"; - foreach ($values as $value) { - if (!$value instanceof Value) { - throw new InvalidArgumentException('Value object expected.'); - } - $query .= ' ' . $table . '.' . $value->getField() . ' = ' . $value->getValue(); - } - - $query .= " FROM ${table}"; - - foreach ($joins as $join) { - if (!$join instanceof Join) { - throw new InvalidArgumentException('Join object expected.'); - } - $query .= ' INNER JOIN ' . $join->getTable() . ' ' . $join->getAlias() . ' ON ' . $join->getOn(); - } - - return $query; + if (empty($values)) { + return ''; + } + + $query = "UPDATE ${table}"; + foreach ($joins as $join) { + if (!$join instanceof Join) { + throw new InvalidArgumentException('Join object expected.'); + } + $query .= ' INNER JOIN ' . $join->getTable() . ' ' . $join->getAlias() . ' ON ' . $join->getOn(); + } + $query .= ' SET '; + $set = []; + foreach ($values as $value) { + if (!$value instanceof Value) { + throw new InvalidArgumentException('Value object expected.'); + } + $set[] = $table . '.' . $value->getField() . ' = ' . $value->getValue(); + } + $query .= implode(', ', $set); + + return $query; } } diff --git a/src/System/Configuration/FileCollector.php b/src/System/Configuration/FileCollector.php index d9cc3c0..f72608a 100644 --- a/src/System/Configuration/FileCollector.php +++ b/src/System/Configuration/FileCollector.php @@ -26,7 +26,8 @@ class FileCollector 'connections', 'config', 'reduce', - 'environments' + 'environments', + 'update_values' ]; /** From 92f621bbb8aab729a367dd387d733dd65a873d40 Mon Sep 17 00:00:00 2001 From: Prince Antil Date: Wed, 29 Jun 2022 20:33:21 +0530 Subject: [PATCH 07/29] SO-Driver-29: Change of config folders names --- {config => .driver}/anonymize.yaml | 0 {config => .driver}/commands.yaml | 0 {config => .driver}/config.yaml | 0 {config => .driver}/connections.yaml | 0 {config => .driver}/engines.yaml | 0 {config => .driver}/environments.yaml | 0 {config => .driver}/pipelines.yaml | 0 .gitignore | 2 +- README.md | 14 ++++++-------- src/Engines/MySql/Sandbox/Sandbox.php | 2 +- src/System/Configuration/FileCollector.php | 4 ++-- 11 files changed, 10 insertions(+), 12 deletions(-) rename {config => .driver}/anonymize.yaml (100%) rename {config => .driver}/commands.yaml (100%) rename {config => .driver}/config.yaml (100%) rename {config => .driver}/connections.yaml (100%) rename {config => .driver}/engines.yaml (100%) rename {config => .driver}/environments.yaml (100%) rename {config => .driver}/pipelines.yaml (100%) diff --git a/config/anonymize.yaml b/.driver/anonymize.yaml similarity index 100% rename from config/anonymize.yaml rename to .driver/anonymize.yaml diff --git a/config/commands.yaml b/.driver/commands.yaml similarity index 100% rename from config/commands.yaml rename to .driver/commands.yaml diff --git a/config/config.yaml b/.driver/config.yaml similarity index 100% rename from config/config.yaml rename to .driver/config.yaml diff --git a/config/connections.yaml b/.driver/connections.yaml similarity index 100% rename from config/connections.yaml rename to .driver/connections.yaml diff --git a/config/engines.yaml b/.driver/engines.yaml similarity index 100% rename from config/engines.yaml rename to .driver/engines.yaml diff --git a/config/environments.yaml b/.driver/environments.yaml similarity index 100% rename from config/environments.yaml rename to .driver/environments.yaml diff --git a/config/pipelines.yaml b/.driver/pipelines.yaml similarity index 100% rename from config/pipelines.yaml rename to .driver/pipelines.yaml diff --git a/.gitignore b/.gitignore index 8846ec0..d1f52e4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ .phpstorm.meta.php composer.lock /vagrant/ -/config.d/ +/driver/ .DS_Store sample/ diff --git a/README.md b/README.md index 3f2a473..b2cdb5c 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Installing Driver is easy: composer require swiftotter/driver ``` -Configuring Driver is easy. In the folder that contains your `vendor/` folder, create a folder called `config`. +Configuring Driver is easy. In the folder that contains your `vendor/` folder, create a folder called `driver`. First, you need to create an Amazon AWS account. We will be using RDS to perform the data manipulations. It is recommended to create a new IAM user with appropriate permissions to access EC2, RDS and S3 (exact permissions will be coming). @@ -85,7 +85,7 @@ To tag an export with an issue number (or whatever is desired): ## Connection Information -Connection information goes into a folder named `config` or `config.d`. The files that are recognized +Connection information goes into a folder named `driver`. The files that are recognized inside these folders are: * `pipelines.yaml` * `commands.yaml` @@ -103,10 +103,8 @@ stored in `/var/www/`. Your vendor directory is `/var/www/vendor/` and, of cours `/var/www/vendor/swiftotter/driver`. As such, Driver will look in the following locations for configuration files: -* `/var/www/config/` -* `/var/www/config.d/` -* `/var/www/vendor/*/*/config/` -* `/var/www/vendor/*/*/config.d/` +* `/var/www/driver/` +* `/var/www/vendor/*/*/.driver/` You can symlink any file you want here. Keep in mind that these files do contain sensitive information and it is necessary to include a `.htaccess` into that folder: @@ -245,7 +243,7 @@ pipelines: ### Creating a new pipeline -The following is taken from `config/pipelines.yaml`. You can put this code in any of the `yaml` files that Driver reads. Just ensure that the +The following is taken from `driver/pipelines.yaml`. You can put this code in any of the `yaml` files that Driver reads. Just ensure that the `pipelines` root node has no space in front of it (exactly as shown below). ```yaml @@ -324,7 +322,7 @@ environments: ### Anonymizing Data -Tables can be anonyimized by creating `anonymize.yaml` in `config/`. The following type of anonymization entities are available in order to provide realistic data and types: +Tables can be anonyimized by creating `anonymize.yaml` in `driver/`. The following type of anonymization entities are available in order to provide realistic data and types: * `email` * `company` diff --git a/src/Engines/MySql/Sandbox/Sandbox.php b/src/Engines/MySql/Sandbox/Sandbox.php index 0e142bb..8e20604 100755 --- a/src/Engines/MySql/Sandbox/Sandbox.php +++ b/src/Engines/MySql/Sandbox/Sandbox.php @@ -378,7 +378,7 @@ private function getAwsParameters($type, $version) ]; if (empty($parameters['region'])) { - $this->output->writeln('No region specified. Are you sure that config.d/connections.yaml exists?'); + $this->output->writeln('No region specified. Are you sure that driver/connections.yaml exists?'); } return $parameters; diff --git a/src/System/Configuration/FileCollector.php b/src/System/Configuration/FileCollector.php index d9cc3c0..3f351c3 100644 --- a/src/System/Configuration/FileCollector.php +++ b/src/System/Configuration/FileCollector.php @@ -15,8 +15,8 @@ class FileCollector { private const FILE_EXTENSION = '.yaml'; private const ALLOWED_FOLDERS = [ - 'config', - 'config.d' + 'driver', + '.driver' ]; private const ALLOWED_FILES = [ 'anonymize', From 7dcaa77c54bcecd4aaeb09229986cc0dffd51771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Thu, 30 Jun 2022 10:05:26 +0200 Subject: [PATCH 08/29] #24 - Smarter anonymization --- config/anonymize.yaml | 74 ++-------------- .../MySql/Transformation/Anonymize.php | 86 +++++-------------- .../MySql/Transformation/Anonymize/Seed.php | 3 - .../MySql/Transformation/UpdateValues.php | 14 +-- 4 files changed, 34 insertions(+), 143 deletions(-) diff --git a/config/anonymize.yaml b/config/anonymize.yaml index 3171730..ea7c236 100644 --- a/config/anonymize.yaml +++ b/config/anonymize.yaml @@ -6,9 +6,6 @@ anonymize: street: 123 Main Street city: Kansas City postcode: 66073 - region: Missouri - country_id: US - region_id: 36 phone: 202-555-0109 ip: 240.36.159.75 @@ -17,9 +14,6 @@ anonymize: street: 2475 Cinnamon Lane city: San Antonio postcode: 78201 - region: Texas - country_id: US - region_id: 57 phone: 202-555-0156 company: Superior Products ip: 181.226.219.191 @@ -29,9 +23,6 @@ anonymize: street: 3931 Jessie Street city: Arabia postcode: 45688 - region: Ohio - country_id: US - region_id: 47 phone: 202-555-0184 company: SwipeWire ip: 78.223.172.48 @@ -41,9 +32,6 @@ anonymize: street: 184 James Street city: New York postcode: 14450 - region: New York - country_id: US - region_id: 12 phone: 202-555-0135 company: SecureSmarter ip: 203.54.176.154 @@ -53,9 +41,6 @@ anonymize: street: 4914 Goff Avenue city: Plainwell postcode: 49080 - region: West Virginia - country_id: US - region_id: 63 phone: 202-555-0115 company: Dwellsmith ip: 51.113.119.34 @@ -65,9 +50,6 @@ anonymize: street: 565 Sampson Street city: Albany postcode: 53502 - region: New York - region_id: 43 - country_id: US phone: 202-555-0190 company: SalePush ip: 85.127.2.228 @@ -77,9 +59,6 @@ anonymize: street: 1763 Davis Avenue city: Sacramento postcode: 95814 - region: California - country_id: US - region_id: 12 phone: (244) 949-1729 company: Cloudrevel ip: 147.199.158.244 @@ -89,9 +68,6 @@ anonymize: street: 451 Wildwood Street city: Akron postcode: 43308 - region: Ohio - country_id: US - region_id: 47 phone: 907-555-0187 company: Crowdstage, Inc. ip: 152.56.2.51 @@ -101,9 +77,6 @@ anonymize: street: 3180 Dancing Dove Lane city: New York postcode: 10004 - region: New York - country_id: US - region_id: 43 phone: 907-555-0192 company: Acme Brands, Inc. ip: 53.243.127.225 @@ -113,9 +86,6 @@ anonymize: street: 2255 Red Maple Drive city: Alhambra postcode: 91801 - region: Oregon - country_id: US - region_id: 49 phone: 404-555-0175 company: QuickSpace ip: 174.146.43.47 @@ -125,9 +95,6 @@ anonymize: street: 1367 Lowndes Hill Park Road city: Sherman Oaks postcode: 91403 - region: Washington - country_id: US - region_id: 62 phone: 404-555-0162 company: Rentoor, inc. ip: 173.8.26.43 @@ -137,9 +104,6 @@ anonymize: street: 674 Cemetery Street city: Magnolia postcode: 61336 - region: Illinois - country_id: US - region_id: 23 phone: 307-555-0141 company: Jumpsync Products, Inc. ip: 68.65.218.25 @@ -149,9 +113,6 @@ anonymize: street: 3357 Zappia Drive city: Lexington postcode: 40507 - region: Kentucky - country_id: US - region_id: 27 phone: 307-555-0126 company: VisionSwipe, Inc. ip: 50.151.191.15 @@ -161,9 +122,6 @@ anonymize: street: 3477 Lake Floyd Circle city: Patuxent Naval Air Test c postcode: 20670 - region: Maryland - country_id: US - region_id: 31 phone: 307-555-0127 company: Shipplier Supplies, Inc. ip: 87.172.86.188 @@ -173,9 +131,6 @@ anonymize: street: 1445 Thompson Street city: Seal Beach postcode: 90740 - region: California - country_id: US - region_id: 12 phone: 515-555-0164 company: TechTack LLC ip: 213.56.195.24 @@ -185,9 +140,6 @@ anonymize: street: 1247 Heliport Loop city: Bloomington postcode: 47408 - region: Illinois - country_id: US - region_id: 23 phone: 515-555-0142 company: Digimail For You, LLC ip: 124.9.254.185 @@ -197,9 +149,6 @@ anonymize: street: 2462 Pritchard Court city: Rio postcode: 26755 - region: West Virginia - country_id: US - region_id: 63 phone: 515-555-0102 company: Kaboom Fireworks ip: 194.59.122.241 @@ -209,9 +158,6 @@ anonymize: street: 1767 Stroop Hill Road city: Fort Wayne postcode: 46852 - region: Indiana - region_id: 24 - country_id: US phone: 860-555-0172 company: Semicolon Bookstore ip: 68.162.14.52 @@ -221,9 +167,6 @@ anonymize: street: 4474 Brookview Drive city: San Jose postcode: 95113 - region: California - region_id: 12 - country_id: US phone: 860-555-0178 company: Bent Out of Shape Jewelry ip: 25.19.163.174 @@ -234,9 +177,6 @@ anonymize: street: 720 Scenic Way city: Warrensburg postcode: 62573 - region: Missouri - region_id: 36 - country_id: US phone: 860-555-0131 ip: 132.60.238.89 @@ -245,7 +185,6 @@ anonymize: street: 55, place de Miremond city: VILLENEUVE-SUR-LOT postcode: 47300 - country_id: FR company: 9Yards Media phone: 05.48.01.14.91 ip: 38.28.94.247 @@ -255,7 +194,6 @@ anonymize: street: 69, boulevard Aristide Briand city: Le Bouscat postcode: 33110 - country_id: FR company: Titan Alarm Group phone: 05.45.43.97.63 ip: 87.122.192.251 @@ -265,7 +203,6 @@ anonymize: street: 59, Avenue des Pr'es city: MONTIGNY-LÈS-METZ postcode: 57158 - country_id: FR phone: 03.31.34.92.91 company: Pacific Alarm Group ip: 171.120.59.109 @@ -275,7 +212,6 @@ anonymize: street: Verlengde Koolhoverweg 10 city: Bocholtz postcode: 6351 - country_id: NL phone: 06-58223374 company: Saga Builders ip: 249.230.160.198 @@ -285,7 +221,15 @@ anonymize: street: De Zaalsteden 70 city: Gieten postcode: 9461 - country_id: NL phone: 06-40996488 company: Drama Studio ip: 49.87.75.212 + + - firstname: Michal + lastname: Biarda + street: 1004 Rabbit Str + city: Lunapark + postcode: 31415 + phone: 852-555-0129 + company: Hello World, Inc. + ip: 197.45.92.201 diff --git a/src/Engines/MySql/Transformation/Anonymize.php b/src/Engines/MySql/Transformation/Anonymize.php index 9411866..ea322eb 100644 --- a/src/Engines/MySql/Transformation/Anonymize.php +++ b/src/Engines/MySql/Transformation/Anonymize.php @@ -1,29 +1,26 @@ configuration = $configuration; $this->properties = $properties; $this->connection = $connection; - $this->logger = $logger; $this->output = $output; $this->seed = $seed; parent::__construct('mysql-transformation-anonymize'); } - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { $config = $this->configuration->getNode('anonymize'); - if (isset($config['disabled']) && $config['disabled'] === true) { - return $transport->withStatus(new Status('mysql-transformation-anonymize', 'success')); - } - - if (!isset($config['tables'])) { - return $transport->withStatus(new Status('mysql-transformation-anonymize', 'success')); - } - - if (!isset($config['seed'])) { + if ((isset($config['disabled']) && $config['disabled'] === true) + || !isset($config['tables']) + || !isset($config['seed']) + ) { return $transport->withStatus(new Status('mysql-transformation-anonymize', 'success')); } @@ -78,7 +68,12 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm return $transport->withStatus(new Status('mysql-transformation-anonymize', 'success')); } - private function anonymize(string $table, array $columns) + public function getProperties(): array + { + return $this->properties; + } + + private function anonymize(string $table, array $columns): void { $connection = $this->connection->getConnection(); @@ -90,35 +85,21 @@ private function anonymize(string $table, array $columns) } catch (\Exception $ex) {} } - foreach ($columns as $columnName => $description) { try { $method = $this->getTypeMethod($description); $select = $this->$method($description['type'] ?? 'general', $columnName, $table); - $query = "UPDATE `${table}` SET `${columnName}` = ${select} WHERE `${table}`.`${columnName}` IS NOT NULL;"; + $query = "UPDATE `${table}` SET `${columnName}` = " + . "${select} WHERE `${table}`.`${columnName}` IS NOT NULL;"; - $connection->beginTransaction(); $connection->query($query); - $connection->commit(); } catch (\Exception $ex) { - $connection->rollBack(); + // Do nothing } } } - /* - * email - * firstname - * lastname - * full_name - * general - * phone - * postcode - * street - * address - */ - /** * Many thanks to this method for inspiration: * https://github.com/DivanteLtd/anonymizer/blob/master/lib/anonymizer/model/database/column.rb @@ -126,14 +107,8 @@ private function anonymize(string $table, array $columns) private function queryEmail(string $type, string $columnName): string { $salt = $this->seed->getSalt(); - return "CONCAT(MD5(CONCAT(\"${salt}\", ${columnName})), \"@\", SUBSTRING({$columnName}, LOCATE('@', {$columnName}) + 1))"; - } - - private function queryFullName(): string - { - $table = Seed::FAKE_USER_TABLE; - - return "(SELECT CONCAT_WS(' ', ${table}.firstname, ${table}.lastname) FROM ${table} ORDER BY RAND() LIMIT 1)"; + return "CONCAT(MD5(CONCAT(\"${salt}\", ${columnName})), \"@\", SUBSTRING({$columnName}, " + . "LOCATE('@', {$columnName}) + 1))"; } private function queryGeneral(string $type, string $columnName, string $mainTable): string @@ -145,14 +120,8 @@ private function queryGeneral(string $type, string $columnName, string $mainTabl $table = Seed::FAKE_USER_TABLE; $salt = $this->seed->getSalt(); $count = $this->seed->getCount(); - return "(SELECT ${table}.${type} FROM ${table} WHERE ${table}.id = (SELECT 1 + MOD(ORD(MD5(CONCAT(\"${salt}\", ${mainTable}.${columnName}))), ${count})) LIMIT 1)"; - } - - private function queryAddress(): string - { - $table = Seed::FAKE_USER_TABLE; - return "(SELECT CONCAT(${table}.street, ', ', ${table}.city, ', ', ${table}.region, ' ', " . - "${table}.postcode, ', ', ${table}.country_id) FROM ${table} ORDER BY RAND() LIMIT 1)"; + return "(SELECT ${table}.${type} FROM ${table} WHERE ${table}.id = " + . "(SELECT 1 + MOD(ORD(MD5(CONCAT(\"${salt}\", ${mainTable}.${columnName}))), ${count})) LIMIT 1)"; } private function queryEmpty(): string @@ -160,16 +129,7 @@ private function queryEmpty(): string return '""'; } - public function getProperties() - { - return $this->properties; - } - - /** - * @param $description - * @return string - */ - private function getTypeMethod($description): string + private function getTypeMethod(array $description): string { $type = $description['type'] ?? 'general'; if ($type === "full_name") { diff --git a/src/Engines/MySql/Transformation/Anonymize/Seed.php b/src/Engines/MySql/Transformation/Anonymize/Seed.php index 386d8b5..cfecc98 100644 --- a/src/Engines/MySql/Transformation/Anonymize/Seed.php +++ b/src/Engines/MySql/Transformation/Anonymize/Seed.php @@ -83,10 +83,7 @@ private function createTable() company VARCHAR(200), street VARCHAR(200), city VARCHAR(200), - region VARCHAR(200), - region_id VARCHAR(10), postcode VARCHAR(200), - country_id VARCHAR(2), phone VARCHAR(200), ip VARCHAR(200) ); diff --git a/src/Engines/MySql/Transformation/UpdateValues.php b/src/Engines/MySql/Transformation/UpdateValues.php index 9d4e163..88920d4 100644 --- a/src/Engines/MySql/Transformation/UpdateValues.php +++ b/src/Engines/MySql/Transformation/UpdateValues.php @@ -13,7 +13,6 @@ use Driver\Pipeline\Transport\Status; use Driver\Pipeline\Transport\TransportInterface; use Driver\System\Configuration; -use Driver\System\Logs\LoggerInterface; use Exception; use InvalidArgumentException; use Symfony\Component\Console\Command\Command; @@ -26,7 +25,6 @@ class UpdateValues extends Command implements CommandInterface { private Configuration $configuration; private RemoteConnectionInterface $connection; - private LoggerInterface $logger; private ConsoleOutput $output; private QueryBuilder $queryBuilder; private array $properties = []; @@ -34,14 +32,12 @@ class UpdateValues extends Command implements CommandInterface public function __construct( Configuration $configuration, RemoteConnectionInterface $connection, - LoggerInterface $logger, ConsoleOutput $output, QueryBuilder $queryBuilder, array $properties = [] ) { $this->configuration = $configuration; $this->connection = $connection; - $this->logger = $logger; $this->output = $output; $this->queryBuilder = $queryBuilder; $this->properties = $properties; @@ -51,11 +47,7 @@ public function __construct( public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { $config = $this->configuration->getNode('update-values'); - if (isset($config['disabled']) && $config['disabled'] === true) { - return $transport->withStatus(new Status('mysql-transformation-update-values', 'success')); - } - - if (!isset($config['tables'])) { + if ((isset($config['disabled']) && $config['disabled'] === true) || !isset($config['tables'])) { return $transport->withStatus(new Status('mysql-transformation-update-values', 'success')); } @@ -87,11 +79,9 @@ private function update(string $table, array $data): void } $connection = $this->connection->getConnection(); try { - $connection->beginTransaction(); $connection->query($query); - $connection->commit(); } catch (Exception $ex) { - $connection->rollBack(); + // Do nothing } } From 05f6fd7ddd827330e962c81345167d1ad8fd60ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Thu, 30 Jun 2022 10:31:03 +0200 Subject: [PATCH 09/29] #22 - PersistentPDO renamed to ReconnectingPDO --- src/Engines/ConnectionInterface.php | 2 +- src/Engines/ConnectionTrait.php | 6 +++--- src/Engines/MySql/Transformation.php | 4 ++-- src/Engines/{PersistentPDO.php => ReconnectingPDO.php} | 2 +- src/System/LocalConnectionLoader.php | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) rename src/Engines/{PersistentPDO.php => ReconnectingPDO.php} (99%) diff --git a/src/Engines/ConnectionInterface.php b/src/Engines/ConnectionInterface.php index 51ff076..2dc054e 100644 --- a/src/Engines/ConnectionInterface.php +++ b/src/Engines/ConnectionInterface.php @@ -23,7 +23,7 @@ interface ConnectionInterface { public function isAvailable(): bool; - public function getConnection(): PersistentPDO; + public function getConnection(): ReconnectingPDO; public function getDSN(): string; diff --git a/src/Engines/ConnectionTrait.php b/src/Engines/ConnectionTrait.php index 7d0c860..37f3c00 100644 --- a/src/Engines/ConnectionTrait.php +++ b/src/Engines/ConnectionTrait.php @@ -8,9 +8,9 @@ trait ConnectionTrait { - private ?PersistentPDO $connection = null; + private ?ReconnectingPDO $connection = null; - public function getConnection(): PersistentPDO + public function getConnection(): ReconnectingPDO { if (!$this->connection) { $options = [ @@ -19,7 +19,7 @@ public function getConnection(): PersistentPDO PDO::ATTR_EMULATE_PREPARES => false ]; - $this->connection = new PersistentPDO($this->getDSN(), $this->getUser(), $this->getPassword(), $options); + $this->connection = new ReconnectingPDO($this->getDSN(), $this->getUser(), $this->getPassword(), $options); } return $this->connection; diff --git a/src/Engines/MySql/Transformation.php b/src/Engines/MySql/Transformation.php index 1c344de..6d84201 100644 --- a/src/Engines/MySql/Transformation.php +++ b/src/Engines/MySql/Transformation.php @@ -21,7 +21,7 @@ use Driver\Commands\CommandInterface; use Driver\Engines\MySql\Sandbox\Utilities; -use Driver\Engines\PersistentPDO; +use Driver\Engines\ReconnectingPDO; use Driver\Engines\RemoteConnectionInterface; use Driver\Pipeline\Environment\EnvironmentInterface; use Driver\Pipeline\Transport\Status; @@ -72,7 +72,7 @@ public function getProperties() return $this->properties; } - private function applyTransformationsTo(PersistentPDO $connection, $transformations) + private function applyTransformationsTo(ReconnectingPDO $connection, $transformations) { array_walk($transformations, function ($query) use ($connection) { try { diff --git a/src/Engines/PersistentPDO.php b/src/Engines/ReconnectingPDO.php similarity index 99% rename from src/Engines/PersistentPDO.php rename to src/Engines/ReconnectingPDO.php index 875e293..b8bc2fb 100644 --- a/src/Engines/PersistentPDO.php +++ b/src/Engines/ReconnectingPDO.php @@ -34,7 +34,7 @@ * @method array|false pgsqlGetNotify(int $fetchMode = 0, int $timeoutMilliseconds = 0) * @method int pgsqlGetPid() */ -class PersistentPDO +class ReconnectingPDO { private const MYSQL_GENERAL_ERROR_CODE = 'HY000'; private const SERVER_HAS_GONE_AWAY_ERROR_CODE = 2006; diff --git a/src/System/LocalConnectionLoader.php b/src/System/LocalConnectionLoader.php index 0bad9b4..559879e 100644 --- a/src/System/LocalConnectionLoader.php +++ b/src/System/LocalConnectionLoader.php @@ -10,7 +10,7 @@ use DI\Container; use Driver\Engines\ConnectionInterface; use Driver\Engines\LocalConnectionInterface; -use Driver\Engines\PersistentPDO; +use Driver\Engines\ReconnectingPDO; class LocalConnectionLoader implements LocalConnectionInterface { @@ -36,7 +36,7 @@ public function __construct( $this->container = $container; } - public function getConnection(): PersistentPDO + public function getConnection(): ReconnectingPDO { return $this->get()->getConnection(); } From 87f9293af6fcfec5be594e13883f50581f47b540 Mon Sep 17 00:00:00 2001 From: Prince Antil Date: Thu, 30 Jun 2022 14:08:07 +0530 Subject: [PATCH 10/29] SO-Driver-29: Replaced references of driver with .driver --- .gitignore | 2 +- README.md | 10 +++++----- src/Engines/MySql/Sandbox/Sandbox.php | 2 +- src/System/Configuration/FileCollector.php | 1 - 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index d1f52e4..c5758ce 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ .phpstorm.meta.php composer.lock /vagrant/ -/driver/ +/.driver/ .DS_Store sample/ diff --git a/README.md b/README.md index b2cdb5c..f1f5e42 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Installing Driver is easy: composer require swiftotter/driver ``` -Configuring Driver is easy. In the folder that contains your `vendor/` folder, create a folder called `driver`. +Configuring Driver is easy. In the folder that contains your `vendor/` folder, create a folder called `.driver`. First, you need to create an Amazon AWS account. We will be using RDS to perform the data manipulations. It is recommended to create a new IAM user with appropriate permissions to access EC2, RDS and S3 (exact permissions will be coming). @@ -85,7 +85,7 @@ To tag an export with an issue number (or whatever is desired): ## Connection Information -Connection information goes into a folder named `driver`. The files that are recognized +Connection information goes into a folder named `.driver`. The files that are recognized inside these folders are: * `pipelines.yaml` * `commands.yaml` @@ -103,7 +103,7 @@ stored in `/var/www/`. Your vendor directory is `/var/www/vendor/` and, of cours `/var/www/vendor/swiftotter/driver`. As such, Driver will look in the following locations for configuration files: -* `/var/www/driver/` +* `/var/www/.driver/` * `/var/www/vendor/*/*/.driver/` You can symlink any file you want here. Keep in mind that these files do contain sensitive information and @@ -243,7 +243,7 @@ pipelines: ### Creating a new pipeline -The following is taken from `driver/pipelines.yaml`. You can put this code in any of the `yaml` files that Driver reads. Just ensure that the +The following is taken from `.driver/pipelines.yaml`. You can put this code in any of the `yaml` files that Driver reads. Just ensure that the `pipelines` root node has no space in front of it (exactly as shown below). ```yaml @@ -322,7 +322,7 @@ environments: ### Anonymizing Data -Tables can be anonyimized by creating `anonymize.yaml` in `driver/`. The following type of anonymization entities are available in order to provide realistic data and types: +Tables can be anonyimized by creating `anonymize.yaml` in `.driver/`. The following type of anonymization entities are available in order to provide realistic data and types: * `email` * `company` diff --git a/src/Engines/MySql/Sandbox/Sandbox.php b/src/Engines/MySql/Sandbox/Sandbox.php index 8e20604..06f2220 100755 --- a/src/Engines/MySql/Sandbox/Sandbox.php +++ b/src/Engines/MySql/Sandbox/Sandbox.php @@ -378,7 +378,7 @@ private function getAwsParameters($type, $version) ]; if (empty($parameters['region'])) { - $this->output->writeln('No region specified. Are you sure that driver/connections.yaml exists?'); + $this->output->writeln('No region specified. Are you sure that .driver/connections.yaml exists?'); } return $parameters; diff --git a/src/System/Configuration/FileCollector.php b/src/System/Configuration/FileCollector.php index 3f351c3..fbda8b2 100644 --- a/src/System/Configuration/FileCollector.php +++ b/src/System/Configuration/FileCollector.php @@ -15,7 +15,6 @@ class FileCollector { private const FILE_EXTENSION = '.yaml'; private const ALLOWED_FOLDERS = [ - 'driver', '.driver' ]; private const ALLOWED_FILES = [ From fba04121218288d6da0a8b2c9341053fe6d14efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Thu, 30 Jun 2022 10:39:38 +0200 Subject: [PATCH 11/29] #23 - mysqldump command upgraded with smarter DEFINER change --- src/Engines/MySql/Export/Primary.php | 4 ++-- src/Engines/MySql/Sandbox/Export.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Engines/MySql/Export/Primary.php b/src/Engines/MySql/Export/Primary.php index c07bb5b..92960aa 100755 --- a/src/Engines/MySql/Export/Primary.php +++ b/src/Engines/MySql/Export/Primary.php @@ -125,7 +125,7 @@ public function assembleEmptyCommand(EnvironmentInterface $environment) [ "--no-data", $tables, - "| sed -e 's/DEFINER[ ]*=[ ]*[^*]*\*/\*/'", + "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g'", ">>", $this->getDumpFile() ] @@ -139,7 +139,7 @@ public function assembleCommand(EnvironmentInterface $environment) [ $this->assembleEmptyTables($environment), $this->assembleIgnoredTables($environment), - "| sed -e 's/DEFINER[ ]*=[ ]*[^*]*\*/\*/'", + "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g'", ">", $this->getDumpFile() ] diff --git a/src/Engines/MySql/Sandbox/Export.php b/src/Engines/MySql/Sandbox/Export.php index bb49e10..582ce46 100755 --- a/src/Engines/MySql/Sandbox/Export.php +++ b/src/Engines/MySql/Sandbox/Export.php @@ -116,7 +116,7 @@ private function assembleCommand($environmentName, $ignoredTables) ], $this->getIgnoredTables($ignoredTables))); $command .= " {$this->connection->getDatabase()} "; - $command .= "| sed -e 's/DEFINER[ ]*=[ ]*[^*]*\*/\*/' "; + $command .= "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' "; if ($this->compressOutput()) { $command .= ' ' . implode(' ', [ From a1889ed5698004ce4a3a890995a636c2ad612d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Thu, 30 Jun 2022 10:46:01 +0200 Subject: [PATCH 12/29] Update .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index c5758ce..e2ef3f6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,5 @@ .phpstorm.meta.php composer.lock /vagrant/ -/.driver/ .DS_Store sample/ From 0aa1c25a482754e6a1df3994d67b7e3f5a25c478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Thu, 30 Jun 2022 11:47:17 +0200 Subject: [PATCH 13/29] #24 - README update --- README.md | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index f1f5e42..1bca767 100644 --- a/README.md +++ b/README.md @@ -87,11 +87,15 @@ To tag an export with an issue number (or whatever is desired): Connection information goes into a folder named `.driver`. The files that are recognized inside these folders are: -* `pipelines.yaml` +* `anonymize.yaml` * `commands.yaml` -* `engines.yaml` -* `connections.yaml` * `config.yaml` +* `connections.yaml` +* `engines.yaml` +* `environments.yaml` +* `pipelines.yaml` +* `reduce.yaml` +* `update_values.yaml` The filenames of these files serve no purpose other than a namespace. The delineation of the configuration happens inside each file. For example, in `pipelines.yaml`, there is a `pipelines` node as the root element. @@ -322,24 +326,19 @@ environments: ### Anonymizing Data -Tables can be anonyimized by creating `anonymize.yaml` in `.driver/`. The following type of anonymization entities are available in order to provide realistic data and types: +Tables can be anonymized by creating `anonymize.yaml` in `.driver/`. The following type of anonymization entities +are available in order to provide realistic data and types: * `email` * `company` * `firstname` * `lastname` -* `full_name` -* `general` * `phone` * `postcode` * `street` -* `address` * `city` -* `region_id` -* `region` -* `country_id` -* `phone` * `ip` +* `general` * `empty` **Example File** @@ -349,8 +348,6 @@ anonymize: quote: customer_email: type: email - customer_name: - type: full_name remote_ip: type: ip ``` From 80d62a1adf56cbfa0d558e8c8022ad802c8d7347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Mon, 4 Jul 2022 10:25:26 +0200 Subject: [PATCH 14/29] #24 - Minor change after code review --- src/Engines/MySql/Transformation/UpdateValues.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Engines/MySql/Transformation/UpdateValues.php b/src/Engines/MySql/Transformation/UpdateValues.php index 88920d4..3822134 100644 --- a/src/Engines/MySql/Transformation/UpdateValues.php +++ b/src/Engines/MySql/Transformation/UpdateValues.php @@ -23,6 +23,8 @@ class UpdateValues extends Command implements CommandInterface { + const COMMAND_NAME = 'mysql-transformation-update-values'; + private Configuration $configuration; private RemoteConnectionInterface $connection; private ConsoleOutput $output; @@ -41,14 +43,14 @@ public function __construct( $this->output = $output; $this->queryBuilder = $queryBuilder; $this->properties = $properties; - parent::__construct('mysql-transformation-update-values'); + parent::__construct(self::COMMAND_NAME); } public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { $config = $this->configuration->getNode('update-values'); if ((isset($config['disabled']) && $config['disabled'] === true) || !isset($config['tables'])) { - return $transport->withStatus(new Status('mysql-transformation-update-values', 'success')); + return $transport->withStatus(new Status(self::COMMAND_NAME, 'success')); } $transport->getLogger()->notice("Beginning table updates from update-values.yaml."); @@ -61,7 +63,7 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm $transport->getLogger()->notice("Database has been updated."); $this->output->writeln("Database has been updated."); - return $transport->withStatus(new Status('mysql-transformation-update-values', 'success')); + return $transport->withStatus(new Status(self::COMMAND_NAME, 'success')); } public function getProperties(): array From b5070650e5b6fa222f8f6d8a262b208c8e3d5736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Mon, 18 Jul 2022 18:18:05 +0200 Subject: [PATCH 15/29] #35 - Added more fake users data, fixed issue of not using all fake users records for anonymization (#36) --- .driver/anonymize.yaml | 135 ++++++++++++++++++ .../MySql/Transformation/Anonymize.php | 7 +- 2 files changed, 140 insertions(+), 2 deletions(-) diff --git a/.driver/anonymize.yaml b/.driver/anonymize.yaml index ea7c236..0dce1be 100644 --- a/.driver/anonymize.yaml +++ b/.driver/anonymize.yaml @@ -233,3 +233,138 @@ anonymize: phone: 852-555-0129 company: Hello World, Inc. ip: 197.45.92.201 + + - firstname: Michael + lastname: Jordan + street: 4182 Grim Avenue + city: Hartford + postcode: '06103' + phone: 860-555-4398 + company: Big Shoe + ip: 71.9.45.231 + + - firstname: Zinedine + lastname: Zidane + street: 4640 Hawks Nest Lane + city: San Diego + postcode: 92103 + phone: 619-555-2688 + company: French Fries + ip: 122.90.87.291 + + - firstname: Bruce + lastname: Lee + street: 3767 Clearview Drive + city: St Louis + postcode: 63101 + phone: 314-555-3035 + company: Karate Kids + ip: 91.23.211.34 + + - firstname: Cameron + lastname: Diaz + street: 3294 Diamond Cove + city: Greenwood Village + postcode: 80111 + phone: 303-555-9124 + company: Nice Movies, Ltd. + ip: 198.34.22.201 + + - firstname: Jennifer + lastname: Lopez + street: 910 Terry Lane + city: Providence + postcode: '02908' + phone: 401-555-9265 + company: Jay-El-Oh + ip: 198.56.81.156 + + - firstname: Brad + lastname: Pitt + street: 14 Atha Drive + city: Orlando + postcode: 32810 + phone: 321-555-9183 + company: Handsome Guys + ip: 19.234.98.125 + + - firstname: John + lastname: Doe + street: 1975 Pinchelone Street + city: Bakersfield + postcode: 93301 + phone: 661-730-4347 + company: Missing People, LLC + ip: 211.50.26.120 + + - firstname: Harlan + lastname: Coben + street: 3182 Nancy Street + city: Durham + postcode: 27713 + phone: 919-555-2387 + company: Myron & Bolitar, Ltd. + ip: 89.57.34.90 + + - firstname: Stephen + lastname: King + street: 4922 Wescam Court + city: Portsmouth + postcode: 23707 + phone: 757-555-1396 + company: IT Lovers + ip: 78.197.204.90 + + - firstname: Will + lastname: Smith + street: 3748 Archwood Avenue + city: Cambridge + postcode: '02141' + phone: 781-555-0073 + company: Men In Black + ip: 38.56.111.78 + + - firstname: Marlon + lastname: Brando + street: 3072 Jadewood Farms + city: Morristown + postcode: '07960' + phone: 973-555-2129 + company: Tea Vee + ip: 50.38.194.29 + + - firstname: Amitkumar + lastname: Solanki + street: 2436 Eden Drive + city: Powell + postcode: 82435 + phone: 307-555-6448 + company: India Express + ip: 163.28.218.109 + + - firstname: Martha + lastname: Stuart + street: 3854 Still Pastures Drive + city: Charles City + postcode: 23030 + phone: 804-555-2583 + company: Cook With Us, LLC + ip: 238.12.1.58 + + - firstname: Arnold + lastname: Schwarzenegger + street: 3869 Beeghley Street + city: Chester + postcode: 29706 + phone: 803-555-2026 + company: Magic Gym + ip: 89.45.149.18 + + - firstname: Bobby + lastname: Dingo + street: 2651 Cecil Street + city: Covington + postcode: 76636 + phone: 254-555-8792 + company: Doggy Barber + ip: 79.3.65.214 diff --git a/src/Engines/MySql/Transformation/Anonymize.php b/src/Engines/MySql/Transformation/Anonymize.php index ea322eb..e84ab99 100644 --- a/src/Engines/MySql/Transformation/Anonymize.php +++ b/src/Engines/MySql/Transformation/Anonymize.php @@ -120,8 +120,11 @@ private function queryGeneral(string $type, string $columnName, string $mainTabl $table = Seed::FAKE_USER_TABLE; $salt = $this->seed->getSalt(); $count = $this->seed->getCount(); - return "(SELECT ${table}.${type} FROM ${table} WHERE ${table}.id = " - . "(SELECT 1 + MOD(ORD(MD5(CONCAT(\"${salt}\", ${mainTable}.${columnName}))), ${count})) LIMIT 1)"; + return "(SELECT ${table}.${type} FROM ${table} WHERE ${table}.id = (SELECT 1 + MOD(" + . "ORD(SUBSTRING(MD5(CONCAT(\"${salt}\", ${mainTable}.${columnName})), 1, 1)) + " + . "ORD(SUBSTRING(MD5(CONCAT(\"${salt}\", ${mainTable}.${columnName})), 2, 1)) + " + . "ORD(SUBSTRING(MD5(CONCAT(\"${salt}\", ${mainTable}.${columnName})), 3, 1)) * " + . "ORD(SUBSTRING(MD5(CONCAT(\"${salt}\", ${mainTable}.${columnName})), 4, 1)), ${count})) LIMIT 1)"; } private function queryEmpty(): string From 0baf2741d811a29e3cd048cfbb7617e95c1d7c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Thu, 21 Jul 2022 12:19:45 +0200 Subject: [PATCH 16/29] #25 - Documentation enhancements, changed default pipeline name to `build` and removed duplicated pipeline configuration --- .driver/config.yaml | 4 +- .driver/connections.yaml.dist | 43 +++ .driver/connections.yaml.example | 24 ++ .driver/environments.yaml.dist | 14 + .driver/environments.yaml.example | 76 ++++++ .driver/pipelines.yaml | 51 +--- LICENSE | 201 -------------- LICENSE.MD | 177 +++++++++++++ README.md | 418 ++++++++++-------------------- composer.json | 6 +- docs/aws-setup.md | 44 ++++ docs/development.md | 124 +++++++++ src/Pipeline/Master.php | 51 ++-- 13 files changed, 662 insertions(+), 571 deletions(-) create mode 100644 .driver/connections.yaml.dist create mode 100644 .driver/connections.yaml.example create mode 100644 .driver/environments.yaml.dist create mode 100644 .driver/environments.yaml.example delete mode 100644 LICENSE create mode 100644 LICENSE.MD create mode 100644 docs/aws-setup.md create mode 100644 docs/development.md diff --git a/.driver/config.yaml b/.driver/config.yaml index 44ba3ab..a7fd7f9 100644 --- a/.driver/config.yaml +++ b/.driver/config.yaml @@ -1,2 +1,4 @@ +configuration: + compress-output: true test: - value: unknown \ No newline at end of file + value: unknown diff --git a/.driver/connections.yaml.dist b/.driver/connections.yaml.dist new file mode 100644 index 0000000..f4c30db --- /dev/null +++ b/.driver/connections.yaml.dist @@ -0,0 +1,43 @@ +connections: + database: mysql # Currently, this is the only supported engine. + webhooks: # OPTIONAL + post-url: # When the process is complete, Driver will ping this URL. + transform-url: # During the transformation process, Driver will ping this URL with connection information. + # You could write your own scripts to be executed at this URL. + auth: + user: # For HTTP basic authentication + password: # For HTTP basic authentication + mysql: # Source database connection information + database: # REQUIRED: database name + charset: # defaults to utf8 + engine: mysql + port: # defaults to 3306 + host: # defaults to 127.0.0.1 + user: # REQUIRED: database username + password: # REQUIRED: database password + dump-path: # Where to put the dumps while they are transitioning between the server and RDS, defaults to /tmp + s3: + engine: s3 + key: # REQUIRED: your S3 login key (can be the same as RDS if both access policies are allowed) + secret: # REQUIRED: your S3 login secret + bucket: # REQUIRED: which bucket would like this dumped into? + region: # defaults to us-east-1 + compressed-file-key: # name in the bucket for a compressed file, e.g. dump{{environment}}.sql.gz + uncompressed-file-key: # name for an uncompressed file, e.g. dump{{environment}}.sql + # It is recommended to include {{environment}} in the filename to avoid multiple environments overwriting the file. + rds: + key: # REQUIRED: your RDS login key + secret: # REQUIRED: your RDS login secret + region: #defaults to us-east-1 + ## BEGIN NEW RDS INSTANCE: + instance-type: # defaults to db.t3.medium, other options can be found at https://aws.amazon.com/rds/instance-types/ + engine: # defaults to MySQL + storage-type: # defaults to gp2 + ## END NEW RDS INSTANCE + ## BEGIN EXISTING RDS INSTANCE: + instance-identifier: + instance-username: + instance-password: + instance-db-name: + security-group-name: + ## END EXISTING RDS INSTANCE diff --git a/.driver/connections.yaml.example b/.driver/connections.yaml.example new file mode 100644 index 0000000..6b80437 --- /dev/null +++ b/.driver/connections.yaml.example @@ -0,0 +1,24 @@ +connections: + database: mysql + mysql: + engine: mysql + host: localhost + database: some_db + user: some_user + password: "sdf93mSDF,os,of3Spwer/df" + s3: + engine: s3 + key: SDF83K0N3IDSL29ZMEIH + secret: "kF9kwnf,mo3fFoaow+3mda/439a,Dlwommw9021m" + bucket: some-bucket + compressed-file-key: dump{{environment}}.sql.gz + uncompressed-file-key: dump{{environment}}.sql + region: us-east-2 + rds: + key: SDF83K0N3IDSL29ZMEIH + secret: "kF9kwnf,mo3fFoaow+3mda/439a,Dlwommw9021m" + region: us-east-2 + ec2: + key: SDF83K0N3IDSL29ZMEIH + secret: "kF9kwnf,mo3fFoaow+3mda/439a,Dlwommw9021m" + region: us-east-2 diff --git a/.driver/environments.yaml.dist b/.driver/environments.yaml.dist new file mode 100644 index 0000000..a24ee19 --- /dev/null +++ b/.driver/environments.yaml.dist @@ -0,0 +1,14 @@ +environments: + ENVIRONMENT_NAME: # Change ENVIRONMENT_NAME to some concrete name like "local", you can define multiple environments. + sort: # REQUIRED: numeric value, lower runs sooner, higher runs later. + transformations: # OPTIONAL + TABLE_NAME: # Change TABLE_NAME to real name of database table you would like to transform with SQL query. + # You can define multiple tables. + - "UPDATE {{table_name}} SET value = 'example' WHERE path = 'id';" # You can list here multiple UPDATE queries. + # The {{table_name}} is substituted for the `TABLE_NAME` reference above. + # Driver will look for a table that ends with `TABLE_NAME`. + # For example, if your `TABLE_NAME` is `core_config_data`, Driver will search for a table in the database that + # ends with `core_config_data`. Thus, `core_config_data` and `sample_core_config_data` would all match. + ignored_tables: # OPTIONAL, tables listed here will be ignored in the final dump with: + # mysqldump ... --ignored-tables=DATABASE.table_1 + - TABLE_NAME diff --git a/.driver/environments.yaml.example b/.driver/environments.yaml.example new file mode 100644 index 0000000..833d7dd --- /dev/null +++ b/.driver/environments.yaml.example @@ -0,0 +1,76 @@ +environments: + staging: + sort: 100 + transformations: + mview_state: + - "UPDATE {{table_name}} SET mode = 'disabled';" + core_config_data: + - "UPDATE {{table_name}} SET value = 'https://staging.store.com/' WHERE path LIKE 'web/%secure/base_url' AND scope_id = 0;" + - "UPDATE {{table_name}} SET value = 'staging.store.com' WHERE path LIKE 'web/cookie/cookie_domain' AND scope_id = 0;" + local: + sort: 200 + transformations: + admin_user: + - "DELETE FROM {{table_name}};" + - "ALTER TABLE {{table_name}} AUTO_INCREMENT = 1;" + - "INSERT INTO {{table_name}} (firstname, lastname, email, username, password) VALUES ('SwiftOtter', 'Studios', 'joseph@swiftotter.com', 'admin', CONCAT(SHA2('xxxxxxxxswiftotter123', 256), ':xxxxxxxx:1'));" + authorization_role: + - "DELETE FROM {{table_name}};" + - "ALTER TABLE {{table_name}} AUTO_INCREMENT = 1;" + - "INSERT INTO {{table_name}} (role_id, parent_id, tree_level, sort_order, role_type, user_id, user_type, role_name) VALUES (1, '0','1','1','G','0','2','Administrators');" + - "INSERT INTO {{table_name}} (role_id, parent_id, tree_level, sort_order, role_type, user_id, user_type, role_name) VALUES (2, '1','2','0','U','1','2','swiftotter');" + authorization_rule: + - "DELETE FROM {{table_name}};" + - "ALTER TABLE {{table_name}} AUTO_INCREMENT = 1;" + - "INSERT INTO {{table_name}} (role_id, resource_id, permission) VALUES ('1', 'Magento_Backend::all', 'allow');" + - "INSERT INTO {{table_name}} (role_id, resource_id, permission) VALUES ('2', 'Magento_Backend::all', 'allow');" + core_config_data: + - "UPDATE {{table_name}} SET value = 'https://store.local/' WHERE path LIKE 'web/%secure/base_url' AND scope_id = 0;" + - "UPDATE {{table_name}} SET value = 'store.local' WHERE path LIKE 'web/cookie/cookie_domain' AND scope_id = 0;" + - "UPDATE {{table_name}} SET value = 'localhost' WHERE path LIKE 'catalog/search/elasticsearch%_server_hostname' AND scope_id = 0;" + ignored_tables: + - setup_module + - customer_address_entity + - customer_address_entity_datetime + - customer_address_entity_decimal + - customer_address_entity_int + - customer_address_entity_text + - customer_address_entity_varchar + - customer_entity + - customer_entity_datetime + - customer_entity_decimal + - customer_entity_int + - customer_entity_text + - customer_entity_varchar + - sales_creditmemo + - sales_credimemo_comment + - sales_creditmemo_grid + - sales_creditmemo_item + - sales_invoice + - sales_invoice_comment + - sales_invoice_grid + - sales_invoice_item + - sales_order + - sales_order_address + - sales_order_grid + - sales_order_item + - sales_order_payment + - sales_order_status_history + - sales_shipment + - sales_shipment_comment + - sales_shipment_grid + - sales_shipment_item + - sales_shipment_track + - sales_invoiced_aggregated + - sales_invoiced_aggregated_order + - sales_payment_transaction + - sales_order_aggregated_created + - sales_order_tax + - sales_order_tax_item + - sales_quote + - sales_quote_address + - sales_quote_address_item + - sales_quote_item + - sales_quote_item_option + - sales_quote_payment + - sales_quote_shipping_rate diff --git a/.driver/pipelines.yaml b/.driver/pipelines.yaml index 786916b..497fd0f 100755 --- a/.driver/pipelines.yaml +++ b/.driver/pipelines.yaml @@ -7,55 +7,8 @@ # at specific environments (like staging and local). pipelines: - default: # pipeline span, usually default is the only one - - name: setup # pipeline stage - sort: 100 - actions: - - name: connect - sort: 100 - - name: check-filesystem - sort: 200 - - name: start-sandbox - sort: 300 - - - name: import - sort: 200 - actions: - - name: export-data-from-system-primary - sort: 100 - - name: import-data-into-sandbox - sort: 200 - - - name: global-commands - sort: 300 - actions: - - name: reduce - sort: 100 - - name: anonymize - sort: 200 - - name: update-values - sort: 300 - - - name: repeat-commands - sort: 400 - actions: - - name: run-transformations - sort: 100 - - name: connect - sort: 200 - - name: export-data-from-sandbox - sort: 300 - - name: upload-data-to-s3 - sort: 400 - - - name: shutdown - sort: 600 - actions: - - name: shutdown-sandbox - sort: 100 - - tag: # pipeline span, usually default is the only one - - name: setup # pipeline stage + build: + - name: setup sort: 100 actions: - name: connect diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 8dada3e..0000000 --- a/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/LICENSE.MD b/LICENSE.MD new file mode 100644 index 0000000..1f51529 --- /dev/null +++ b/LICENSE.MD @@ -0,0 +1,177 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md index 1bca767..3fd23a8 100644 --- a/README.md +++ b/README.md @@ -1,92 +1,64 @@ # Driver -### A database task-runner +Driver is a database task-runner that helps you to turn production database into a staging/development database sandbox. + +## Description Driver is the easy way to turn a production database into a staging or development-friendly database. It is built with the aim of complete flexibility. Additionally, with the built-in capability of working with RDS, Driver can run transformations on database host that is completely separate from your production environment. -Because Driver is a task-runner, there is a good chance that you will need to create some tasks to run. -There are some additional modules out there to help you with this. You will also need to specify configuration. - -## TL;DR +Driver resides on your production machine, with your website's codebase (via Composer). +You set up a CRON job to run Driver. -Driver resides on your production machine, preferrably with your website's codebase (via composer). You setup a cron job to run Driver. - -* **Driver connects to your production database ONCE via `mysqldump`.** +* Driver connects to your production database ONCE via `mysqldump`. * Driver dumps that to a file on your system. -* Driver creates a RDS instance and pushes the database dump up to RDS (via **https**). +* Driver creates an RDS instance and pushes the database dump up to RDS (via https). * Driver runs whatever actions you would like (configured globally or per environment). * Driver dumps the transformed data, zips it and pushes it up to S3. +* You can then download transformed dump from S3 to your staging/development machine, also using Driver. -For a 3-5GB database, this process could take an hour or two. This depends on how many environments you are creating and what type of RDS instance you are using. This causes no downtime to the frontend thanks to the `--single-transaction` flag on `mysqldump`. Yes, it does take a while to run, but there is little tono impact on the frontend. +For a 3-5GB database, this process could take an hour or two. +This depends on how many environments you are creating and what type of RDS instance you are using. +This causes no downtime to the frontend thanks to the `--single-transaction` flag on `mysqldump`. +Yes, it does take a while to run, but there is little-to-no impact on the frontend. -## Quickstart +## Before We Start -Downloading a Driver-created file is quite simple: -```bash -./vendor/bin/driver run --environment=local [or whatever environment that's been configured] import-s3 -``` +At Swift Otter we mainly work with Magento 2. We created separate repo for Magento 2 Driver transformations. +It includes config for anonymization of all database tables of core Magento 2 system. If you are Magento 2 developer, +we recommend you to install that repo, instead of this one (Driver would be installed anyway, as it is a dependency +of that Magento 2 repo). If you're not a Magento 2 developer, we still recommend you to look at that repo to get +a better idea on how things can be configured for your project. -Installing Driver is easy: -``` -composer require swiftotter/driver -``` +Magento 2 Driver transformations repo can be found [here](https://github.com/swiftotter/Driver-Magento2). -Configuring Driver is easy. In the folder that contains your `vendor/` folder, create a folder called `.driver`. -First, you need to create an Amazon AWS account. We will be using RDS to perform the data manipulations. It is -recommended to create a new IAM user with appropriate permissions to access EC2, RDS and S3 (exact permissions -will be coming). - -Place inside it a file with the following information (replacing all of the brackets and their content): -```yaml -# -connections: - database: mysql - mysql: - engine: mysql - database: [DATABASE_NAME] - user: [DATABASE_USER] - password: [DATABASE_PASSWORD] - s3: - engine: s3 - key: [IAM_KEY] - secret: [IAM_KEY] - bucket: [YOUR_BUCKET] - compressed-file-key: sample.gz - uncompressed-file-key: sample.sql - region: us-east-1 - rds: - key: [IAM_KEY] - secret: [IAM_KEY] - region: us-east-1 - ec2: - key: [IAM_KEY] - secret: [IAM_KEY] - region: us-east-1 -``` +## Getting Started -It should run! Be prepared for it to take some time (on the order of hours). -``` -./vendor/bin/driver run -``` +### Dependencies -You can run a specific pipeline. If you use the `default` pipeline, then you don't have to specify the pipeline. Included is a `tag` pipeline to generate one database snapshot. -```bash -./vendor/bin/driver run tag - /\ environment name -``` +- PHP 7.4 or higher + +### Prerequisites + +You need to create an Amazon AWS account. We will be using RDS to perform the data manipulations. +It is recommended to create a new IAM user with appropriate permissions to access EC2, RDS and S3 +(exact permissions will be coming). This is explained in details [here](docs/aws-setup.md). + +### Installation -To tag an export with an issue number (or whatever is desired): ```bash -./vendor/bin/driver run tag --tag=DEV-1234 +composer require swiftotter/driver ``` -## Connection Information +### Configuration + +#### Overview of Config Structure + +Driver configuration goes into a folder named `.driver`. +The files that are recognized inside this folder are: -Connection information goes into a folder named `.driver`. The files that are recognized -inside these folders are: * `anonymize.yaml` * `commands.yaml` * `config.yaml` @@ -110,223 +82,37 @@ files: * `/var/www/.driver/` * `/var/www/vendor/*/*/.driver/` -You can symlink any file you want here. Keep in mind that these files do contain sensitive information and -it is necessary to include a `.htaccess` into that folder: -`Deny from all` - -## AWS Setup - -Driver's default implementation is to use AWS for the database transformation (RDS) and the storage of the transformed -databases (S3). - -You will need to do two things in your AWS control panel: -1. Create a new policy. -2. Assign that policy to a new user. - -### Policy Creation - -Open your control panel and go to IAM. Click on the Policies tab on the sidebar. Choose to Create New Policy. -Select Create Your Own Policy (if you want to use the one below) and enter the following code. - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ec2:AuthorizeSecurityGroupIngress", - "ec2:CreateSecurityGroup", - "s3:GetObject", - "s3:PutObject", - "rds:CreateDBInstance", - "rds:DeleteDBInstance", - "rds:DescribeDBInstances" - ], - "Resource": [ - "*" - ] - } - ] -} -``` - -### User Creation - -In the IAM control panel, click on the Users tab. Select Add user. Choose a username. This will only be seen by you -in the control panel. Check the Programmatic access as Driver will be needing a access key ID and a secret access key. -Select Add existing policies directly and choose your newly-created policy. Review it and then create the user. - -Place the Access key ID and Secret access key in your configuration. - - -### Connection Reference - -```yaml -configuration: - compress-output: true # if set, the output will be compressed and the compressed-file-key will be used. - -connections: - database: mysql # Currently, this is the only supported engine. - webhooks: - post-url: https://whatever-your-site-is.com # When the process is complete, Driver will ping this url. - transform-url: # During the transformation process, Driver will ping this url with connection information. - # You could write your own scripts to be executed at this url. - auth: - user: # for HTTP basic authentication - password: # for HTTP basic authentication - mysql: # Source database connection information - database: your_database_name # REQUIRED - charset: # defaults to utf8 - engine: mysql - port: # defaults to 3306 - host: # defaults to 127.0.0.1 - user: # REQUIRED: database username - password: # REQUIRED: database password - dump-path: /tmp # Where to put the dumps while they are transitioning between the server and RDS - s3: - engine: s3 - key: # REQUIRED: your S3 login key (can be the same as RDS if both access policies are allowed) - secret: # REQUIRED: your S3 login secret - bucket: # REQUIRED: which bucket would like this dumped into? - region: # defaults to us-east-1 - compressed-file-key: # name in the bucket for a compressed file. - uncompressed-file-key: # name for an uncompressed file. - # It is recommended to include {{environment}} in the filename to avoid multiple environments overwriting the file. - rds: - key: # REQUIRED: your RDS login key - secret: # REQUIRED: your RDS login secret - region: #defaults to us-east-1 - ## BEGIN NEW RDS INSTANCE: - instance-type: # REQUIRED: choose from left column in https://aws.amazon.com/rds/details/#DB_Instance_Classes - engine: MySQL - storage-type: gp2 - ## END NEW RDS INSTANCE - ## BEGIN EXISTING RDS INSTANCE: - instance-identifier: - instance-username: - instance-password: - instance-db-name: - security-group-name: - ## END EXISTING RDS INSTANCE -``` - -### Terminology - -* **Pipeline** (specified in `pipelines.yaml`): a series of stages. Representative of the work to take a production database and transform it. These are found under the `pipelines` node. You can execute a specific pipeline with the `--pipeline` argument in the command line. -* **Stage**: groups of actions inside of a pipeline. Stages are run sequentially Right now, Driver does work this way, but actions could run in parallel. -* **Action**: Specific command to run. - -### Modifying a existing pipeline - -`default` is the "default" pipeline. This may be all you need. The next section will talk about the syntax for creating a new pipeline. - -Here is an example of adding a custom action to the `default` pipeline: - -```yaml -commands: - reset-admin-password: - class: \YourApplication\Driver\Transformations\ResetAdminPassword - # ensure that: - # 1) the composer autoloader can find this class - # 2) your class implements \Driver\Commands\CommandInterface - # 3) it preferably extends \Symfony\Component\Console\Command\Command - -pipelines: - default: - - name: global-commands - # you can add stages or use an existing one. global-commands runs - # after the data has been pushed into RDS and before any transformations - # run. Keep in mind, you can create a new stage prefixed with "repeat-" - # and it will run it once per environment. - actions: - - name: reset-admin-password - # notice this name matches the name we added in commands - sort: 100 -``` - -### Creating a new pipeline - -The following is taken from `.driver/pipelines.yaml`. You can put this code in any of the `yaml` files that Driver reads. Just ensure that the -`pipelines` root node has no space in front of it (exactly as shown below). - -```yaml -pipelines: # root node - YOUR_PIPELINE_NAME: # pipeline span name - - name: setup # pipeline stage name - sort: 100 # sort order - actions: # stages / actions to run - - name: connect - sort: 100 - - name: check-filesystem - sort: 200 - - name: start-sandbox - sort: 300 - - - name: import - sort: 200 - actions: - - name: export-data-from-system-primary - sort: 100 - - name: import-data-into-sandbox - sort: 200 - - - name: global-commands - sort: 300 - actions: - - name: empty - sort: 1 - - - name: repeat-commands - sort: 400 - actions: - - name: run-transformations - sort: 1000 - - - name: repeat-export - sort: 400 - actions: - - name: export-data-from-sandbox - sort: 100 - - name: upload-data-to-s3 - sort: 200 - - - name: shutdown - sort: 500 - actions: - - name: shutdown-sandbox - sort: 100 - -``` +You can symlink any file you want here. Keep in mind that these files do contain sensitive information, and +it is necessary to include a `.htaccess` into that folder: `Deny from all` +#### Connections Configuration -### Environments +You have to configure connections information for your project. +In the folder that contains your `vendor` folder, create a folder called `.driver`. +Next, copy the file `vendor/swiftotter/driver/.driver/connections.yaml.dist` to `.driver/connections.yaml` +and fill it in with your source MySQL information, as well as destination EC2, RDS and S3 data. -Ok, so Driver sounds really cool to create you a staging database. But, what about a database for the devs? There is a few changes for that: we want to clear out all other admin users (and reset the remaining admin user's password) as well as set unique urls for the website and possibly other things. +Exemplary config can be found in `vendor/swiftotter/driver/.driver/connections.yaml.example`. -There is a solution for that, too. Environments allow you to easily make modifications to the database, per-site. +#### Environments Configuration -**Note:** you can execute specific commands per environment. These changes do not revert for each environment, but rather they are applied according to their sort order. If one environment uses the data in the `admin_users` table, and another environment clears out all data in the `admin_users` table, you set the sort order for the first table lower (ex. `100`) and the sort order for the second table higher (ex. `200`). +Environments allow to make modifications to the database, per-site. You can configure different set of actions for every +environment. For example, for local environment you may want to clear out all admin users as well as set unique URLs +for the website and possibly other things. +**Note:** You can execute specific commands per environment. These changes do not revert for each environment, +but rather they are applied according to their sort order. If one environment uses the data in the `admin_users` table, +and another environment clears out all data in the `admin_users` table, you set the sort order for the first environment +lower (ex. `100`) and the sort order for the second environment higher (ex. `200`). -#### Environment Reference: +To configure your environments copy the file `vendor/swiftotter/driver/.driver/environments.yaml.dist` +to `.driver/environments.yaml` and fill it in with your environments data. -```yaml -environments: - ENVIRONMENT_NAME: - sort: # lower runs sooner, higher runs later - transformations: - TABLE_NAME: - - "UPDATE {{table_name}} SET value = 'test-value' WHERE path = 'id';" - ignored_tables: - # These are ignored in the final dump: mysqldump ... --ignored-tables=DATABASE.table_1 - - table_1 - - table_2 -``` +Exemplary config can be found in `vendor/swiftotter/driver/.driver/environments.yaml.example`. -### Anonymizing Data +#### Anonymization Configuration -Tables can be anonymized by creating `anonymize.yaml` in `.driver/`. The following type of anonymization entities +Tables can be anonymized by creating `anonymize.yaml` file in `.driver/`. The following type of anonymization entities are available in order to provide realistic data and types: * `email` @@ -344,22 +130,86 @@ are available in order to provide realistic data and types: **Example File** ``` anonymize: - tables: - quote: - customer_email: - type: email - remote_ip: - type: ip + tables: + quote: + customer_email: + type: email + remote_ip: + type: ip +``` + +#### Post-Anonymization Values Updates Configuration + +Sometimes in your database you can store in a single field some value that was originally built by concatenating +fields from different tables, eg. `full_name` or `shipping_address`. Driver allows to run some specific values updates +after anonymization of simple fields is finished. Thanks to this feature you can anonymize such complex fields too. + +To create post-anonymization values updates, create `update_values.yaml` file in `.driver/`. + +**Example File** +``` +update-values: + tables: + customer_grid_flat: + joins: + - table: customer_entity + alias: c + on: c.entity_id = customer_grid_flat.entity_id + - table: customer_address_entity + alias: ba + on: ba.entity_id = c.default_billing + - table: customer_address_entity + alias: sa + on: sa.entity_id = c.default_shipping + values: + - field: name + value: CONCAT_WS(' ', c.firstname, c.lastname) + - field: shipping_full + value: CONCAT(sa.street, ', ', sa.city, ', ', IFNULL(sa.region, '-'), ' ', sa.postcode, ', ', sa.country_id) + - field: billing_full + value: CONCAT(ba.street, ', ', ba.city, ', ', IFNULL(ba.region, '-'), ' ', ba.postcode, ', ', ba.country_id) +``` + +## Usage + +### Building Database Sandboxes For All Environments + +Be prepared for it to take some time (on the order of hours). + +``` +./vendor/bin/driver run +``` + +### Building Database Sandbox For Single Environment + +```bash +./vendor/bin/driver run build --environment=envname +``` + +where `envname` is one of the environments defined in `environments.yaml` config file. + +### Tagging Database Sandbox + +Sandboxes built with a tag will include the tag in the file name. + +```bash +./vendor/bin/driver run build --tag=tagname ``` +where `tagname` is an issue number (or whatever is desired). -**Notes:** -* The `{{table_name}}` is substituted for the `TABLE_NAME` reference above. Driver will look for a table that **ends** with `TABLE_NAME`. For example, if your `TABLE_NAME` is `core_config_data`, Driver will search for a table in the database that ends with `core_config_data`. Thus, `mage_core_config_data`, `sample_core_config_data` and `core_config_data` would all match. +### Download and Import Sandbox Database from S3 to staging/local environment -### -#### Download and Import S3 uploaded database on STAGING/Local environment: +The below command run the export/import pipeline which download the database into your project `var/` directory, +create new database in MySQL, and import the downloaded database in it for your environment. -The below command run the export/import pipeline which download the database into your project var/ directory, create new database in mysql, and import the downloaded database in it for your environment. ``` -./vendor/bin/driver run import-s3 +./vendor/bin/driver run import-s3 --environment=envname --tag=tagname ``` + +`tag` is optional. + +## Contribution + +We welcome you to contribute in Driver development. You can find some handy information for developers +[here](docs/development.md). diff --git a/composer.json b/composer.json index 19b2bb2..6d5504a 100755 --- a/composer.json +++ b/composer.json @@ -1,8 +1,10 @@ { "name": "swiftotter/driver", - "description": "A database production to sandbox utility to sanitize data.", + "description": "A database task-runner that helps you to turn production database into a staging/development database sandbox", + "license": "Apache-2.0", "type": "library", "require": { + "php": "^7.4|^8.0", "ext-pdo": "*", "php-di/php-di": "^6.0", "aws/aws-sdk-php": "^3.19", @@ -24,7 +26,7 @@ }, "authors": [ { - "name": "Joseph Maxwell", + "name": "SwiftOtter, Inc.", "email": "joseph@swiftotter.com" } ], diff --git a/docs/aws-setup.md b/docs/aws-setup.md new file mode 100644 index 0000000..18e7934 --- /dev/null +++ b/docs/aws-setup.md @@ -0,0 +1,44 @@ +# AWS Setup + +Driver's default implementation is to use AWS for the database transformation (RDS) and the storage of the transformed +databases (S3). + +You will need to do two things in your AWS control panel: +1. Create a new policy. +2. Assign that policy to a new user. + +## Policy Creation + +Open your control panel and go to IAM. Click on the Policies tab on the sidebar. Choose to Create New Policy. +Select Create Your Own Policy (if you want to use the one below) and enter the following code. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateSecurityGroup", + "s3:GetObject", + "s3:PutObject", + "rds:CreateDBInstance", + "rds:DeleteDBInstance", + "rds:DescribeDBInstances" + ], + "Resource": [ + "*" + ] + } + ] +} +``` + +## User Creation + +In the IAM control panel, click on the Users tab. Select Add user. Choose a username. This will only be seen by you +in the control panel. Check the Programmatic access as Driver will be needing a access key ID and a secret access key. +Select Add existing policies directly and choose your newly-created policy. Review it and then create the user. + +Place the Access key ID and Secret access key in your configuration. diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..0aec4af --- /dev/null +++ b/docs/development.md @@ -0,0 +1,124 @@ +# Development + +If you're a Driver developer, here you will find some handy information that will help you with your work. + +## Terminology + +* **Pipeline** (specified in `.driver/pipelines.yaml`): a series of stages. Representative of the work to take a production database and transform it. These are found under the `pipelines` node. You can execute a specific pipeline with the `--pipeline` argument in the command line. +* **Stage**: groups of actions inside a pipeline. Stages are run sequentially. Right now, Driver does work this way, but actions could run in parallel. +* **Action**: Specific command to run. + +## Modifying an existing pipeline + +`build` is the "default" pipeline. This may be all you need. The next section will talk about the syntax for creating a new pipeline. + +Here is an example of adding a custom action to the `build` pipeline: + +```yaml +commands: + reset-admin-password: + class: \YourApplication\Driver\Transformations\ResetAdminPassword + # ensure that: + # 1) the composer autoloader can find this class + # 2) your class implements \Driver\Commands\CommandInterface + # 3) it preferably extends \Symfony\Component\Console\Command\Command + +pipelines: + build: + - name: global-commands + # you can add stages or use an existing one. global-commands runs + # after the data has been pushed into RDS and before any transformations + # run. Keep in mind, you can create a new stage prefixed with "repeat-" + # and it will run it once per environment. + actions: + - name: reset-admin-password + # notice this name matches the name we added in commands + sort: 100 +``` + +## Creating a new pipeline + +The following is taken from `.driver/pipelines.yaml`. You can put this code in any of the `yaml` files that Driver reads. +Just ensure that the `pipelines` root node has no space in front of it (exactly as shown below). + +```yaml +pipelines: # root node + YOUR_PIPELINE_NAME: # pipeline span name + - name: setup # pipeline stage name + sort: 100 # sort order + actions: # stages / actions to run + - name: connect + sort: 100 + - name: check-filesystem + sort: 200 + - name: start-sandbox + sort: 300 + + - name: import + sort: 200 + actions: + - name: export-data-from-system-primary + sort: 100 + - name: import-data-into-sandbox + sort: 200 + + - name: global-commands + sort: 300 + actions: + - name: empty + sort: 1 + + - name: repeat-commands + sort: 400 + actions: + - name: run-transformations + sort: 1000 + + - name: repeat-export + sort: 400 + actions: + - name: export-data-from-sandbox + sort: 100 + - name: upload-data-to-s3 + sort: 200 + + - name: shutdown + sort: 500 + actions: + - name: shutdown-sandbox + sort: 100 +``` + +## Debug Mode + +Working with AWS RDS is nice but super slow. For development purposes you can configure local database that will act +as RDS. All you need to do is to add the following to your `.driver/connections.yaml` config file: + +```yaml +connections: + mysql_debug: + engine: mysql + host: localhost + database: local_db + username: some_user + password: some_password +``` + +and then run `build` pipeline with `--debug` option, e.g.: + +```shell +./vendor/bin/driver run build --environment=local --debug +``` + +**Watchout!** Debug database should NOT be your local project's database. + +## Increasing Verbosity of Messages + +By default, Driver doesn't output much information to the user. +During development, we recommend to add `-vvv` to all commands you're running, e.g.: + +```shell +./vendor/bin/driver run build --environment=local --debug -vvv +``` + +This will show error and debug information. diff --git a/src/Pipeline/Master.php b/src/Pipeline/Master.php index 80ae513..347fa23 100644 --- a/src/Pipeline/Master.php +++ b/src/Pipeline/Master.php @@ -1,58 +1,41 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/8/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline; -use Driver\Pipeline\Exception\PipeLineNotFound as PipeLineNotFoundException; use Driver\Pipeline\Span\Factory as PipeLineSpanFactory; use Driver\Pipeline\Transport\Factory as TransportFactory; +use Driver\Pipeline\Transport\TransportInterface; use Driver\System\Configuration; class Master { - const EMPTY_NODE = 'empty'; - const DEFAULT_NODE = 'default'; - protected $configuration; - protected $pipeLineFactory; - protected $transportFactory; + const DEFAULT_NODE = 'build'; - public function __construct(Configuration $configuration, PipeLineSpanFactory $pipeLineFactory, TransportFactory $transportFactory) - { + private Configuration $configuration; + private PipeLineSpanFactory $pipeLineFactory; + private TransportFactory $transportFactory; + + public function __construct( + Configuration $configuration, + PipeLineSpanFactory $pipeLineFactory, + TransportFactory $transportFactory + ) { $this->configuration = $configuration; $this->pipeLineFactory = $pipeLineFactory; $this->transportFactory = $transportFactory; } - public function runDefault() - { - $this->run(self::DEFAULT_NODE); - } - - public function run($set) + public function run(string $set): void { $pipeline = $this->pipeLineFactory->create($set); - $transport = $pipeline($this->createTransport()); + $transport = $pipeline($this->createTransport($set)); $pipeline->cleanup($transport); } - protected function createTransport() + protected function createTransport(string $set): TransportInterface { - return $this->transportFactory->create(self::DEFAULT_NODE); + return $this->transportFactory->create($set); } } From 343fdd5c74289d304c7a5e7746f3952a229ee2a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Tue, 20 Sep 2022 15:49:18 +0200 Subject: [PATCH 17/29] #38: Replaced Mysqli with PDO in import-s3 action --- src/Engines/MySql/Import/Primary.php | 131 +++++++++++---------------- 1 file changed, 55 insertions(+), 76 deletions(-) diff --git a/src/Engines/MySql/Import/Primary.php b/src/Engines/MySql/Import/Primary.php index b29592b..5e16599 100755 --- a/src/Engines/MySql/Import/Primary.php +++ b/src/Engines/MySql/Import/Primary.php @@ -27,7 +27,8 @@ use Driver\System\Configuration; use Driver\System\LocalConnectionLoader; use Driver\System\Logs\LoggerInterface; -use Driver\System\Random; +use PDO; +use PDOException; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Output\ConsoleOutput; @@ -42,34 +43,22 @@ class Primary extends Command implements CommandInterface /** @var LoggerInterface */ private $logger; - /** @var Random */ - private $random; - - /** @var ?string */ - private $path; - /** @var Configuration */ private $configuration; /** @var ConsoleOutput */ private $output; - private $preserved; - - const DEFAULT_DUMP_PATH = '/tmp'; - public function __construct( LocalConnectionLoader $localConnection, Configuration $configuration, LoggerInterface $logger, - Random $random, ConsoleOutput $output, array $properties = [] ) { $this->localConnection = $localConnection; $this->properties = $properties; $this->logger = $logger; - $this->random = $random; $this->configuration = $configuration; $this->output = $output; return parent::__construct('import-data-from-system-primary'); @@ -81,10 +70,14 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm $this->output->writeln("Import database from var/ into local MySQL started"); $this->output->writeln("Preparing MySQL Connection using Magento (app/etc/env.php)"); - $conn = mysqli_connect($this->localConnection->getHost(), $this->localConnection->getUser(), $this->localConnection->getPassword()); - - if (!$conn) { - $this->output->writeln('Could not connect: ' . mysqli_connect_error() . ''); + try { + $conn = new PDO( + "mysql:host={$this->localConnection->getHost()}", + $this->localConnection->getUser(), + $this->localConnection->getPassword() + ); + } catch (PDOException $e) { + $this->output->writeln('Could not connect: ' . $e->getMessage() . ''); $this->output->write([ ' Connected with:', 'Host:' . $this->localConnection->getHost(), @@ -92,20 +85,17 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm 'Password:' . preg_replace('/.*/i', '*', $this->localConnection->getPassword()), '' ]); - throw new \Exception('Could not connect: ' . mysqli_connect_error()); + throw new \Exception('Could not connect: ' . $e->getMessage()); } $this->output->writeln("Creating Local Database: " . $this->getDatabaseCommand($environment) ); - mysqli_query($conn, $this->getDatabaseCommand($environment)); - if ($conn->error !== "" && (strpos($conn->error, "database exists") === false)) { - $this->output->writeln('Database cannot be created: ' . $conn->error . ''); + if (!$conn->query($this->getDatabaseCommand($environment))) { + $this->output->writeln('Database cannot be created: ' . $conn->errorInfo()[2] . ''); } - mysqli_close($conn); - $transport->getLogger()->debug( "Local connection string: " . str_replace( $this->localConnection->getPassword(), @@ -132,7 +122,9 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm $this->output->writeln('Import to local MYSQL completed.'); $this->restore($preserved); - $this->output->writeln('Rows were inserted/updated back into ' . implode(', ', array_keys($preserved)) . '.'); + $this->output->writeln( + 'Rows were inserted/updated back into ' . implode(', ', array_keys($preserved)) . '.' + ); return $transport->withStatus(new Status('db_import', 'success')); } @@ -172,42 +164,35 @@ private function preserve(): array $output = []; - try { - $preserve = $this->localConnection->getPreserve(); - - // I hate this cyclomatic complexity, but it's the most reasonable solution for this depth of configuration. - foreach ($preserve as $tableName => $columns) { - foreach ($columns as $columnName => $selectData) { - foreach ($selectData as $like) { - $preparedTableName = mysqli_real_escape_string($connection, $tableName); - $preparedColumnName = mysqli_real_escape_string($connection, $columnName); - $tableColumnNames = $this->getColumns($tableName); - $columnNames = $this->flattenColumns($tableColumnNames); - - if (!count($tableColumnNames)) { + $preserve = $this->localConnection->getPreserve(); + + // I hate this cyclomatic complexity, but it's the most reasonable solution for this depth of configuration. + foreach ($preserve as $tableName => $columns) { + foreach ($columns as $columnName => $selectData) { + foreach ($selectData as $like) { + $tableColumnNames = $this->getColumns($tableName); + $columnNames = $this->flattenColumns($tableColumnNames); + + if (!count($tableColumnNames)) { + continue; + } + + try { + $statement = $connection->prepare("SELECT ${columnNames} FROM ${tableName} " + . "WHERE ${columnName} LIKE :like"); + if ($statement === false) { continue; } - try { - $stmt = $connection->prepare("SELECT ${columnNames} FROM ${preparedTableName} WHERE ${preparedColumnName} LIKE ?"); - if ($stmt === false) { - continue; - } - - $stmt->bind_param("s", $like); - $stmt->execute(); - $result = $stmt->get_result(); - while ($row = $result->fetch_array(MYSQLI_ASSOC)) { - $output[$tableName][] = $row; - } - } catch (\Exception $ex) { - continue; + $statement->execute(['like' => $like]); + while ($row = $statement->fetch(PDO::FETCH_ASSOC)) { + $output[$tableName][] = $row; } + } catch (\Exception $ex) { + continue; } } } - } finally { - $connection->close(); } return $output; @@ -218,22 +203,18 @@ private function getColumns($tableName): array $connection = $this->getConnection(); $columns = []; - try { - $result = $connection->query("SHOW COLUMNS FROM ${tableName};"); - if (!$result) { - return []; - } - - while ($row = $result->fetch_assoc()) { - if (isset($row['Extra']) - && $row['Extra'] === 'auto_increment') { - continue; - } + $statement = $connection->prepare("SHOW COLUMNS FROM ${tableName}"); + if (!$statement->execute()) { + return []; + } - $columns[] = $row['Field']; + while ($row = $statement->fetch(PDO::FETCH_ASSOC)) { + if (isset($row['Extra']) + && $row['Extra'] === 'auto_increment') { + continue; } - } finally { - $connection->close(); + + $columns[] = $row['Field']; } return $columns; @@ -250,7 +231,6 @@ private function restore(array $values): void foreach ($rows as $row) { $connection = $this->getConnection(); - $preparedTableName = mysqli_real_escape_string($connection, $tableName); $tableColumnNames = $this->getColumns($tableName); $columnNames = $this->flattenColumns($tableColumnNames); $columnFillers = implode(', ', array_fill(0, count($row), '?')); @@ -258,20 +238,19 @@ private function restore(array $values): void return "`${key}` = VALUES(`${key}`)"; }, array_keys($row))); - $stmt = $connection->prepare("INSERT INTO ${preparedTableName} (${columnNames}) VALUES(${columnFillers}) ON DUPLICATE KEY UPDATE ${valuesList}"); - $stmt->bind_param(implode('', array_fill(0, count($row), 's')), ...array_values($row)); - $stmt->execute(); + $statement = $connection->prepare("INSERT INTO ${tableName} (${columnNames}) VALUES(${columnFillers})" + ." ON DUPLICATE KEY UPDATE ${valuesList}"); + $statement->execute(array_values($row)); } } } - private function getConnection() + private function getConnection(): PDO { - return mysqli_connect( - $this->localConnection->getHost(), + return new PDO( + "mysql:host={$this->localConnection->getHost()};dbname={$this->localConnection->getDatabase()}", $this->localConnection->getUser(), - $this->localConnection->getPassword(), - $this->localConnection->getDatabase() + $this->localConnection->getPassword() ); } } From 3442bc9d5b9d8f91fadb7b72a2ca05fc349b6092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Wed, 21 Sep 2022 10:48:34 +0200 Subject: [PATCH 18/29] #39: Added configuration option for RDS engine version (#41) * #39: Added configuration option for RDS engine version * #39: Removed default RDS engine version * #39: Really removed default RDS engine version --- .driver/connections.yaml | 2 +- .driver/connections.yaml.dist | 1 + .driver/connections.yaml.example | 2 ++ src/Engines/MySql/Sandbox/Sandbox.php | 23 ++++++++++++++--------- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.driver/connections.yaml b/.driver/connections.yaml index 0297c66..09188f7 100644 --- a/.driver/connections.yaml +++ b/.driver/connections.yaml @@ -4,4 +4,4 @@ connections: engine: MySQL storage-type: gp2 -# TODO: fix merging so that the other keys overwrite this. \ No newline at end of file +# TODO: fix merging so that the other keys overwrite this. diff --git a/.driver/connections.yaml.dist b/.driver/connections.yaml.dist index f4c30db..785947a 100644 --- a/.driver/connections.yaml.dist +++ b/.driver/connections.yaml.dist @@ -32,6 +32,7 @@ connections: ## BEGIN NEW RDS INSTANCE: instance-type: # defaults to db.t3.medium, other options can be found at https://aws.amazon.com/rds/instance-types/ engine: # defaults to MySQL + engine-version: # defaults to AWS RDS default engine version, possible versions can be found by using console command `aws rds describe-db-engine-versions` storage-type: # defaults to gp2 ## END NEW RDS INSTANCE ## BEGIN EXISTING RDS INSTANCE: diff --git a/.driver/connections.yaml.example b/.driver/connections.yaml.example index 6b80437..6e094d5 100644 --- a/.driver/connections.yaml.example +++ b/.driver/connections.yaml.example @@ -18,6 +18,8 @@ connections: key: SDF83K0N3IDSL29ZMEIH secret: "kF9kwnf,mo3fFoaow+3mda/439a,Dlwommw9021m" region: us-east-2 + engine: MySQL + engine-version: 5.7.38 ec2: key: SDF83K0N3IDSL29ZMEIH secret: "kF9kwnf,mo3fFoaow+3mda/439a,Dlwommw9021m" diff --git a/src/Engines/MySql/Sandbox/Sandbox.php b/src/Engines/MySql/Sandbox/Sandbox.php index 06f2220..3c83804 100755 --- a/src/Engines/MySql/Sandbox/Sandbox.php +++ b/src/Engines/MySql/Sandbox/Sandbox.php @@ -29,6 +29,8 @@ class Sandbox { + const DEFAULT_ENGINE = 'MySQL'; + private $configuration; private $instance; private $initialized; @@ -36,16 +38,15 @@ class Sandbox private $logger; private $random; private $awsClientFactory; - private $securityGroupId; private $securityGroupName; - private $dbName; private $identifier; private $username; private $password; private $statuses; private $output; + /** @var \Symfony\Component\EventDispatcher\EventDispatcher */ private EventDispatcher $eventDispatcher; @@ -102,6 +103,11 @@ public function init() 'StorageType' => $this->getStorageType() ]; + $engineVersion = $this->getEngineVersion(); + if ($engineVersion) { + $parameters['EngineVersion'] = $engineVersion; + } + if ($parameterGroupName = $this->getDbParameterGroupName()) { $parameters['DBParameterGroupName'] = $parameterGroupName; } @@ -286,15 +292,14 @@ private function getStorageType() return $storageType; } - private function getEngine() + private function getEngine(): string { - $engine = $this->configuration->getNode('connections/rds/engine'); - - if (!$engine) { - $engine = 'MySQL'; - } + return $this->configuration->getNode('connections/rds/engine') ?? self::DEFAULT_ENGINE; + } - return $engine; + private function getEngineVersion(): ?string + { + return $this->configuration->getNode('connections/rds/engine-version'); } public function getSecurityGroupName() From cf94a3d7fd855ee5eb1fc5c2c80b23b7e41adc4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Thu, 6 Oct 2022 17:13:31 +0200 Subject: [PATCH 19/29] #42: PHP 8.1 compatibility fixes (#44) * #42: PHP 8.1 compatibility fixes, unit tests updates, codebase cleanup with PHPCS, improved error handling, several minor fixes * #42: Micro fixes for PHP 7.4 compatibility * #42: Micro code optimization after code review --- .driver/commands.yaml | 2 - .driver/connections.yaml | 1 + .gitignore | 1 + composer.json | 13 +- phpcs.xml | 40 +++ phpunit.xml | 9 +- src/Commands/CleanupInterface.php | 27 +- src/Commands/CommandInterface.php | 28 +- src/Commands/Environment/Setup.php | 32 +-- src/Commands/ErrorInterface.php | 25 +- src/Commands/Factory.php | 56 ++-- src/Commands/Webhook/PostCommand.php | 32 +-- src/Commands/Webhook/Transform.php | 48 ++-- src/Commands/Webhook/Webhook.php | 42 ++- src/Commands/Webhook/WebhookInterface.php | 30 +-- src/Engines/ConnectionInterface.php | 20 +- src/Engines/LocalConnectionInterface.php | 20 +- src/Engines/MySql.php | 61 ++--- src/Engines/MySql/Check.php | 74 ++---- src/Engines/MySql/Export/Primary.php | 72 +++-- src/Engines/MySql/Import/Primary.php | 92 +++---- src/Engines/MySql/Sandbox/Connection.php | 29 +- src/Engines/MySql/Sandbox/Export.php | 75 +++--- src/Engines/MySql/Sandbox/Import.php | 55 ++-- src/Engines/MySql/Sandbox/Init.php | 34 +-- src/Engines/MySql/Sandbox/Sandbox.php | 250 +++++++++--------- src/Engines/MySql/Sandbox/Shutdown.php | 35 +-- src/Engines/MySql/Sandbox/Ssl.php | 45 +--- src/Engines/MySql/Sandbox/Utilities.php | 72 ++--- src/Engines/MySql/Transformation.php | 42 ++- .../MySql/Transformation/Anonymize.php | 18 +- .../MySql/Transformation/Anonymize/Seed.php | 13 +- src/Engines/MySql/Transformation/Reduce.php | 17 +- .../MySql/Transformation/UpdateValues.php | 12 +- src/Engines/ReconnectingPDO.php | 7 +- src/Engines/RemoteConnectionInterface.php | 21 +- src/Engines/S3/Download.php | 62 ++--- src/Engines/S3/Upload.php | 48 ++-- src/Pipeline/Command.php | 80 +++--- .../Environment/EnvironmentInterface.php | 84 +----- src/Pipeline/Environment/Factory.php | 61 ++--- src/Pipeline/Environment/Manager.php | 74 +++--- src/Pipeline/Environment/Primary.php | 123 +++------ .../Exception/EnvironmentNotFound.php | 25 -- src/Pipeline/Exception/PipeLineNotFound.php | 22 +- src/Pipeline/Master.php | 5 +- src/Pipeline/PipeInterface.php | 28 -- src/Pipeline/Span/Factory.php | 51 +--- src/Pipeline/Span/Primary.php | 129 ++++----- src/Pipeline/Span/SpanInterface.php | 32 +-- src/Pipeline/Stage/Factory.php | 38 +-- src/Pipeline/Stage/Primary.php | 161 ++++++----- src/Pipeline/Stage/StageInterface.php | 33 +-- src/Pipeline/Transport/Error.php | 35 --- src/Pipeline/Transport/Factory.php | 38 +-- src/Pipeline/Transport/Primary.php | 105 ++++---- src/Pipeline/Transport/Status.php | 37 +-- src/Pipeline/Transport/TransportInterface.php | 86 ++---- src/System/Application.php | 60 +---- src/System/Arguments.php | 39 --- src/System/AwsClientFactory.php | 30 +-- src/System/Configuration.php | 29 +- .../Configuration/FolderCollectionFactory.php | 2 +- src/System/ConnectionLoaderInterface.php | 13 - src/System/DebugExternalConnection.php | 22 +- src/System/DebugMode.php | 7 +- src/System/DefaultConnection.php | 19 +- src/System/DependencyConfig.php | 35 +-- src/System/Entry.php | 38 +-- src/System/Factory/Base.php | 29 -- src/System/Factory/FactoryInterface.php | 25 -- src/System/Factory/PropertyInterface.php | 25 -- src/System/LocalConnectionLoader.php | 25 +- src/System/Logs/LoggerInterface.php | 23 +- src/System/Logs/Primary.php | 90 +++---- src/System/Random.php | 25 +- src/System/RemoteIP.php | 27 +- src/System/S3FilenameFormatter.php | 11 +- src/System/Tag.php | 5 +- src/System/YamlFormatter.php | 72 +---- src/Tests/Unit/Commands/FactoryTest.php | 33 +-- src/Tests/Unit/Engines/MySql/CheckTest.php | 39 --- .../Unit/Engines/MySql/MySqlAbstract.php | 25 -- .../Engines/MySql/Sandbox/SandboxTest.php | 35 +-- src/Tests/Unit/Engines/MySql/dump.xml | 64 ----- src/Tests/Unit/Helper/DI.php | 25 +- src/Tests/Unit/Pipeline/MasterTest.php | 45 ---- src/Tests/Unit/Pipeline/Span/FactoryTest.php | 58 ++-- src/Tests/Unit/Pipeline/Span/PrimaryTest.php | 31 +-- src/Tests/Unit/Pipeline/Stage/FactoryTest.php | 41 +-- src/Tests/Unit/Pipeline/Stage/PrimaryTest.php | 31 +-- .../Unit/Pipeline/Transport/PrimaryTest.php | 69 +++-- src/Tests/Unit/System/ApplicationTest.php | 35 +-- src/Tests/Unit/System/ArgumentsTest.php | 34 --- src/Tests/Unit/System/ConfigurationTest.php | 5 +- src/Tests/Unit/System/FactoryTest.php | 33 --- src/Tests/Unit/Utils.php | 32 +-- 97 files changed, 1306 insertions(+), 2787 deletions(-) create mode 100644 phpcs.xml delete mode 100644 src/Pipeline/Exception/EnvironmentNotFound.php delete mode 100644 src/Pipeline/PipeInterface.php delete mode 100644 src/Pipeline/Transport/Error.php delete mode 100644 src/System/Arguments.php delete mode 100644 src/System/ConnectionLoaderInterface.php delete mode 100644 src/System/Factory/Base.php delete mode 100644 src/System/Factory/FactoryInterface.php delete mode 100644 src/System/Factory/PropertyInterface.php delete mode 100644 src/Tests/Unit/Engines/MySql/CheckTest.php delete mode 100644 src/Tests/Unit/Engines/MySql/MySqlAbstract.php delete mode 100644 src/Tests/Unit/Engines/MySql/dump.xml delete mode 100644 src/Tests/Unit/Pipeline/MasterTest.php delete mode 100755 src/Tests/Unit/System/ArgumentsTest.php delete mode 100755 src/Tests/Unit/System/FactoryTest.php diff --git a/.driver/commands.yaml b/.driver/commands.yaml index 39668a6..9be74a5 100644 --- a/.driver/commands.yaml +++ b/.driver/commands.yaml @@ -1,6 +1,4 @@ commands: - pipeline: - class: \Driver\Pipeline\Command connect: class: \Driver\Engines\MySql check-filesystem: diff --git a/.driver/connections.yaml b/.driver/connections.yaml index 09188f7..82bde8b 100644 --- a/.driver/connections.yaml +++ b/.driver/connections.yaml @@ -3,5 +3,6 @@ connections: instance-type: db.t3.medium engine: MySQL storage-type: gp2 + region: us-east-2 # TODO: fix merging so that the other keys overwrite this. diff --git a/.gitignore b/.gitignore index e2ef3f6..66f8279 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ composer.lock /vagrant/ .DS_Store sample/ +.phpunit.result.cache diff --git a/composer.json b/composer.json index 6d5504a..dcc15d1 100755 --- a/composer.json +++ b/composer.json @@ -6,19 +6,18 @@ "require": { "php": "^7.4|^8.0", "ext-pdo": "*", - "php-di/php-di": "^6.0", "aws/aws-sdk-php": "^3.19", - "symfony/yaml": ">=2.3", - "ericpoe/haystack": "^1.0", "guzzlehttp/guzzle": "^6.2|^7.0", + "php-di/php-di": "^6.0", "psr/log": "^1.0", "symfony/console": "^4.4", - "symfony/event-dispatcher": "^4.0" + "symfony/event-dispatcher": "^4.0", + "symfony/yaml": ">=2.3" }, "require-dev": { - "squizlabs/php_codesniffer": "^2.7", - "phpunit/phpunit": "^5.6", - "phpunit/dbunit": "^2.0" + "phpunit/phpunit": "^9.5", + "slevomat/coding-standard": "^8.5", + "squizlabs/php_codesniffer": "^3.7" }, "bin": ["bin/driver"], "autoload": { diff --git a/phpcs.xml b/phpcs.xml new file mode 100644 index 0000000..0a68e2a --- /dev/null +++ b/phpcs.xml @@ -0,0 +1,40 @@ + + + src + + + 7 + warning + + + 7 + warning + + + 7 + warning + + + 7 + warning + + + 7 + warning + + + + + + + + 7 + warning + + + + 7 + warning + + + diff --git a/phpunit.xml b/phpunit.xml index 6fb9d2d..29f9c71 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -1,10 +1,9 @@ - + src/Tests/Unit/** - \ No newline at end of file + diff --git a/src/Commands/CleanupInterface.php b/src/Commands/CleanupInterface.php index e2e9f79..68950e1 100644 --- a/src/Commands/CleanupInterface.php +++ b/src/Commands/CleanupInterface.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Commands; @@ -24,9 +9,5 @@ interface CleanupInterface { - /** - * @param TransportInterface $transport - * @return TransportInterface - */ - public function cleanup(TransportInterface $transport, EnvironmentInterface $environment); -} \ No newline at end of file + public function cleanup(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface; +} diff --git a/src/Commands/CommandInterface.php b/src/Commands/CommandInterface.php index 9083446..0641273 100644 --- a/src/Commands/CommandInterface.php +++ b/src/Commands/CommandInterface.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Commands; @@ -24,11 +9,8 @@ interface CommandInterface { - /** - * @param TransportInterface $transport - * @return TransportInterface - */ - public function go(TransportInterface $transport, EnvironmentInterface $environment); + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface; - public function getProperties(); + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array; } diff --git a/src/Commands/Environment/Setup.php b/src/Commands/Environment/Setup.php index 33d5e72..659b0c5 100644 --- a/src/Commands/Environment/Setup.php +++ b/src/Commands/Environment/Setup.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/10/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Commands\Environment; @@ -26,22 +11,25 @@ class Setup extends Command implements CommandInterface { - private $properties; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; - public function __construct($properties = []) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function __construct(array $properties = []) { $this->properties = $properties; parent::__construct('environment-configuration'); } - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { + // @todo Check this! return $transport->withEnvironment($environment); } - } diff --git a/src/Commands/ErrorInterface.php b/src/Commands/ErrorInterface.php index 17db47f..0b4ebbf 100644 --- a/src/Commands/ErrorInterface.php +++ b/src/Commands/ErrorInterface.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Commands; @@ -24,9 +9,5 @@ interface ErrorInterface { - /** - * @param TransportInterface $transport - * @return TransportInterface - */ - public function error(TransportInterface $transport, EnvironmentInterface $environment); + public function error(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface; } diff --git a/src/Commands/Factory.php b/src/Commands/Factory.php index 33f6330..22e001e 100644 --- a/src/Commands/Factory.php +++ b/src/Commands/Factory.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Commands; @@ -24,9 +9,10 @@ class Factory { - private $configuration; - private $container; - private $substitutions; + private Configuration $configuration; + private Container $container; + /** @var array|null */ + private ?array $substitutions = null; public function __construct(Configuration $configuration, Container $container) { @@ -34,34 +20,34 @@ public function __construct(Configuration $configuration, Container $container) $this->container = $container; } - /** - * @param $name - * @return CommandInterface - */ - public function create($name, $properties = []) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function create(string $name, array $properties = []): CommandInterface { $className = $this->getClassName($name); return $this->container->make($className, ['properties' => $properties]); } - private function getClassName($name) + private function getClassName(string $name): string { - $class = $this->runSubstitutions($this->configuration->getNode("commands/{$name}/class")); + $class = $this->runSubstitutions((string)$this->configuration->getNode("commands/{$name}/class")); if (class_exists($class) && in_array(CommandInterface::class, class_implements($class))) { return $class; } else { - throw new \Exception("{$name} doesn't exist or it doesn't implement the type " . CommandInterface::class . "."); + var_dump($class); + throw new \Exception( + "{$name} doesn't exist or it doesn't implement the type " . CommandInterface::class . "." + ); } } - private function runSubstitutions($name) + private function runSubstitutions(string $name): string { $substitutions = $this->getSubstitutions(); preg_match_all("/%(.+)%/U", $name, $matches); if (count($matches) > 1) { - $replacements = array_reduce($matches[1], function($carry, $name) use ($substitutions) { - $carry['%'.$name.'%'] = $substitutions[$name]; + $replacements = array_reduce($matches[1], function ($carry, $name) use ($substitutions) { + $carry['%' . $name . '%'] = $substitutions[$name]; return $carry; }, []); @@ -71,7 +57,10 @@ private function runSubstitutions($name) } } - private function getSubstitutions() + /** + * @return string[] + */ + private function getSubstitutions(): array { if (!$this->substitutions) { $databaseEngine = $this->configuration->getNode('connections/database'); @@ -83,9 +72,8 @@ private function getSubstitutions() 'engine' => $this->configuration->getNode("engines/{$databaseEngine}/class-name") ]; $this->substitutions = $substitutions; - } return $this->substitutions; } -} \ No newline at end of file +} diff --git a/src/Commands/Webhook/PostCommand.php b/src/Commands/Webhook/PostCommand.php index 7550b90..b72c46a 100644 --- a/src/Commands/Webhook/PostCommand.php +++ b/src/Commands/Webhook/PostCommand.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/6/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Commands\Webhook; @@ -29,10 +14,12 @@ class PostCommand extends Command implements CommandInterface { - private $configuration; - private $webhook; - private $properties; + private Configuration $configuration; + private Webhook $webhook; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct(Configuration $configuration, Webhook $webhook, array $properties = []) { $this->configuration = $configuration; @@ -42,12 +29,13 @@ public function __construct(Configuration $configuration, Webhook $webhook, arra parent::__construct('webhook-post-command'); } - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { $url = $this->configuration->getNode('connections/webhooks/post-url'); diff --git a/src/Commands/Webhook/Transform.php b/src/Commands/Webhook/Transform.php index 13e6655..b38a1b9 100644 --- a/src/Commands/Webhook/Transform.php +++ b/src/Commands/Webhook/Transform.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/6/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Commands\Webhook; @@ -29,15 +14,21 @@ class Transform extends Command implements CommandInterface { - const ACTION = 'transform'; - - private $configuration; - private $webhook; - private $sandbox; - private $properties; - - public function __construct(Configuration $configuration, Webhook $webhook, Sandbox $sandbox, array $properties) - { + private const ACTION = 'transform'; + + private Configuration $configuration; + private Webhook $webhook; + private Sandbox $sandbox; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; + + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function __construct( + Configuration $configuration, + Webhook $webhook, + Sandbox $sandbox, + array $properties = [] + ) { $this->configuration = $configuration; $this->webhook = $webhook; $this->sandbox = $sandbox; @@ -46,12 +37,13 @@ public function __construct(Configuration $configuration, Webhook $webhook, Sand parent::__construct('webhook-transform-command'); } - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { $url = $this->configuration->getNode('connections/webhooks/transform-url'); diff --git a/src/Commands/Webhook/Webhook.php b/src/Commands/Webhook/Webhook.php index 091ad4d..d9832fb 100644 --- a/src/Commands/Webhook/Webhook.php +++ b/src/Commands/Webhook/Webhook.php @@ -1,32 +1,18 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/6/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Commands\Webhook; use Driver\System\Configuration; use Driver\System\Logs\LoggerInterface; +use GuzzleHttp\Client; use Symfony\Component\Console\Output\OutputInterface; class Webhook implements WebhookInterface { - private $configuration; - private $logger; + private Configuration $configuration; + private LoggerInterface $logger; public function __construct(Configuration $configuration, LoggerInterface $logger) { @@ -34,9 +20,10 @@ public function __construct(Configuration $configuration, LoggerInterface $logge $this->logger = $logger; } - public function call($webhookUrl, $data, $method) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function call(string $webhookUrl, array $data, string $method): void { - $client = new \GuzzleHttp\Client(); + $client = new Client(); $options = $this->getAuth([]); $options = $this->getData($options, $data, $method); @@ -47,17 +34,20 @@ public function call($webhookUrl, $data, $method) } } - public function post($webhookUrl, $data) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function post(string $webhookUrl, array $data): void { $this->call($webhookUrl, $data, 'POST'); } - public function get($webhookUrl, $data) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function get(string $webhookUrl, array $data): void { $this->call($webhookUrl, $data, 'GET'); } - private function getData($options, $data, $method) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification,SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + private function getData(array $options, array $data, string $method): array { if (!count($data)) { return $options; @@ -70,7 +60,8 @@ private function getData($options, $data, $method) } } - private function getAuth($options) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification,SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + private function getAuth(array $options): array { $auth = $this->configuration->getNode('connections/webhook/auth'); $node = []; @@ -81,5 +72,4 @@ private function getAuth($options) return array_merge($options, $node); } - } diff --git a/src/Commands/Webhook/WebhookInterface.php b/src/Commands/Webhook/WebhookInterface.php index 00776ba..fb18902 100644 --- a/src/Commands/Webhook/WebhookInterface.php +++ b/src/Commands/Webhook/WebhookInterface.php @@ -1,29 +1,17 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/6/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Commands\Webhook; interface WebhookInterface { - public function call($webhookUrl, $data, $method); + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function call(string $webhookUrl, array $data, string $method): void; - public function post($webhookUrl, $data); + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function post(string $webhookUrl, array $data): void; - public function get($webhookUrl, $data); -} \ No newline at end of file + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function get(string $webhookUrl, array $data): void; +} diff --git a/src/Engines/ConnectionInterface.php b/src/Engines/ConnectionInterface.php index 2dc054e..1af0a4d 100644 --- a/src/Engines/ConnectionInterface.php +++ b/src/Engines/ConnectionInterface.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/3/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines; @@ -39,5 +24,6 @@ public function getUser(): string; public function getPassword(): string; + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification public function getPreserve(): array; } diff --git a/src/Engines/LocalConnectionInterface.php b/src/Engines/LocalConnectionInterface.php index cfc6770..edf2d08 100644 --- a/src/Engines/LocalConnectionInterface.php +++ b/src/Engines/LocalConnectionInterface.php @@ -1,25 +1,9 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/3/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines; interface LocalConnectionInterface extends ConnectionInterface { - } diff --git a/src/Engines/MySql.php b/src/Engines/MySql.php index 4b0ba39..f7d5c1c 100644 --- a/src/Engines/MySql.php +++ b/src/Engines/MySql.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/19/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines; @@ -30,39 +15,33 @@ class MySql extends Command implements CommandInterface { - /** @var LoggerInterface */ - private $logger; - - /** @var array */ - private $properties; - - /** @var LocalConnectionLoader */ - private $connection; - - /** @var ConsoleOutput */ - private $output; - - public function __construct(LocalConnectionLoader $connection, LoggerInterface $logger, ConsoleOutput $output, array $properties = []) - { + private LocalConnectionLoader $connection; + private LoggerInterface $logger; + private ConsoleOutput $output; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; + + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function __construct( + LocalConnectionLoader $connection, + LoggerInterface $logger, + ConsoleOutput $output, + array $properties = [] + ) { $this->logger = $logger; $this->properties = $properties; $this->connection = $connection; - $this->output = $output; - parent::__construct(null); + $this->output = $output; + parent::__construct(); } - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - protected function configure() - { - $this->setName('mysql-connect') - ->setDescription('Connects to MySQL.'); - } - - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { $value = $this->connection->getConnection()->getAttribute(\PDO::ATTR_CONNECTION_STATUS); $this->output->writeln('Successfully connected: ' . $value . ''); diff --git a/src/Engines/MySql/Check.php b/src/Engines/MySql/Check.php index 411eeef..1687c35 100644 --- a/src/Engines/MySql/Check.php +++ b/src/Engines/MySql/Check.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/19/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines\MySql; @@ -30,33 +15,24 @@ class Check extends Command implements CommandInterface { - /** @var LocalConnectionLoader */ - private $configuration; - - /** @var int */ - private $databaseSize; - - /** @var int */ - private $freeSpace; - - /** @var LoggerInterface */ - private $logger; + private const DB_SIZE_KEY = 'database_size'; - /** @var array */ - private $properties; - - /** @var ConsoleOutput */ - private $output; - - const DB_SIZE_KEY = 'database_size'; + private LocalConnectionLoader $configuration; + private LoggerInterface $logger; + private ConsoleOutput $output; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; + private ?int $databaseSize; + private ?int $freeSpace; + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( LocalConnectionLoader $connection, LoggerInterface $logger, ConsoleOutput $output, array $properties = [], - $databaseSize = null, - $freeSpace = null + ?int $databaseSize = null, + ?int $freeSpace = null ) { $this->configuration = $connection; $this->databaseSize = $databaseSize; @@ -68,45 +44,51 @@ public function __construct( return parent::__construct('mysql-system-check'); } - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { /** @var OutputInterface $output */ if ($this->getDatabaseSize() < $this->getDriveFreeSpace()) { - $this->output->writeln("Database size (" . $this->getDatabaseSize() . " MB) is less than the free space available on the PHP drive."); + $this->output->writeln( + "Database size (" . $this->getDatabaseSize() + . " MB) is less than the free space available on the PHP drive." + ); $this->logger->info("Database size is less than the free space available on the PHP drive."); - return $transport->withNewData(self::DB_SIZE_KEY, $this->getDatabaseSize())->withStatus(new Status('check', 'success')); + return $transport->withNewData(self::DB_SIZE_KEY, $this->getDatabaseSize()) + ->withStatus(new Status('check', 'success')); } else { $this->output->writeln("There is NOT enough free space to dump the database."); throw new \Exception('There is NOT enough free space to dump the database.'); } } - private function getDriveFreeSpace() + private function getDriveFreeSpace(): int { if ($this->freeSpace) { return $this->freeSpace; } else { - return ceil(disk_free_space(getcwd()) / 1024 / 1024); + return (int)ceil(disk_free_space(getcwd()) / 1024 / 1024); } } - private function getDatabaseSize() + private function getDatabaseSize(): int { if ($this->databaseSize) { return $this->databaseSize; } else { $connection = $this->configuration->getConnection(); $statement = $connection->prepare( - 'SELECT ceiling(sum(data_length + index_length) / 1024 / 1024) as DB_SIZE FROM information_schema.tables WHERE table_schema = :database_name GROUP BY table_schema;' + 'SELECT ceiling(sum(data_length + index_length) / 1024 / 1024) as DB_SIZE ' + . 'FROM information_schema.tables WHERE table_schema = :database_name GROUP BY table_schema;' ); $statement->execute(['database_name' => $this->configuration->getDatabase() ]); - return $statement->fetch(\PDO::FETCH_ASSOC)["DB_SIZE"]; + return (int)$statement->fetch(\PDO::FETCH_ASSOC)["DB_SIZE"]; } } } diff --git a/src/Engines/MySql/Export/Primary.php b/src/Engines/MySql/Export/Primary.php index 92960aa..7de4c05 100755 --- a/src/Engines/MySql/Export/Primary.php +++ b/src/Engines/MySql/Export/Primary.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/3/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines\MySql\Export; @@ -26,25 +11,25 @@ use Driver\Pipeline\Transport\Status; use Driver\Pipeline\Transport\TransportInterface; use Driver\System\Configuration; -use Driver\System\LocalConnectionLoader; use Driver\System\Logs\LoggerInterface; use Driver\System\Random; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Output\ConsoleOutput; -use Symfony\Component\Console\Output\OutputInterface; class Primary extends Command implements CommandInterface, CleanupInterface { + private const DEFAULT_DUMP_PATH = '/tmp'; + private LocalConnectionInterface $localConnection; - private array $properties = []; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; private LoggerInterface $logger; private Random $random; private ?string $path = null; private Configuration $configuration; private ConsoleOutput $output; - const DEFAULT_DUMP_PATH = '/tmp'; - + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( LocalConnectionInterface $localConnection, Configuration $configuration, @@ -62,7 +47,7 @@ public function __construct( return parent::__construct('mysql-default-export'); } - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { $transport->getLogger()->notice("Exporting database from local MySql"); $this->output->writeln("Exporting database from local MySql"); @@ -75,13 +60,11 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm ) ); $this->output->writeln("Local connection string: " . str_replace( - $this->localConnection->getPassword(), - '', - $this->assembleCommand($environment) - ) - ); + $this->localConnection->getPassword(), + '', + $this->assembleCommand($environment) + )); - $results = null; $command = implode(';', array_filter([ $this->assembleCommand($environment), $this->assembleEmptyCommand($environment) @@ -95,24 +78,26 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm } else { $this->logger->notice("Database dump has completed."); $this->output->writeln("Database dump has completed."); - return $transport->withStatus(new Status('sandbox_init', 'success'))->withNewData('dump-file', $this->getDumpFile()); + return $transport->withStatus(new Status('sandbox_init', 'success')) + ->withNewData('dump-file', $this->getDumpFile()); } } - public function cleanup(TransportInterface $transport, EnvironmentInterface $environment) + public function cleanup(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { if ($this->getDumpFile() && file_exists($this->getDumpFile())) { @unlink($this->getDumpFile()); } + return $transport; } - - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - public function assembleEmptyCommand(EnvironmentInterface $environment) + public function assembleEmptyCommand(EnvironmentInterface $environment): string { $tables = implode(' ', $environment->getEmptyTables()); @@ -121,7 +106,7 @@ public function assembleEmptyCommand(EnvironmentInterface $environment) } return implode(' ', array_merge( - $this->getDumpCommand($environment), + $this->getDumpCommand(), [ "--no-data", $tables, @@ -132,10 +117,10 @@ public function assembleEmptyCommand(EnvironmentInterface $environment) )); } - public function assembleCommand(EnvironmentInterface $environment) + public function assembleCommand(EnvironmentInterface $environment): string { return implode(' ', array_merge( - $this->getDumpCommand($environment), + $this->getDumpCommand(), [ $this->assembleEmptyTables($environment), $this->assembleIgnoredTables($environment), @@ -146,7 +131,10 @@ public function assembleCommand(EnvironmentInterface $environment) )); } - private function getDumpCommand(EnvironmentInterface $environment) + /** + * @return string[] + */ + private function getDumpCommand(): array { return [ "mysqldump --user=\"{$this->localConnection->getUser()}\"", @@ -159,7 +147,7 @@ private function getDumpCommand(EnvironmentInterface $environment) ]; } - private function assembleEmptyTables(EnvironmentInterface $environment) + private function assembleEmptyTables(EnvironmentInterface $environment): string { $tables = $environment->getEmptyTables(); $output = []; @@ -171,17 +159,17 @@ private function assembleEmptyTables(EnvironmentInterface $environment) return implode(' ', $output); } - private function assembleIgnoredTables(EnvironmentInterface $environment) + private function assembleIgnoredTables(EnvironmentInterface $environment): string { $tables = $environment->getIgnoredTables(); - $output = implode(' | ', array_map(function($table) { + $output = implode(' | ', array_map(function ($table) { return "awk '!/^INSERT INTO `{$table}` VALUES/'"; }, $tables)); return $output ? ' | ' . $output : ''; } - private function getDumpFile() + private function getDumpFile(): string { if (!$this->path) { $path = $this->configuration->getNode('connections/mysql/dump-path'); diff --git a/src/Engines/MySql/Import/Primary.php b/src/Engines/MySql/Import/Primary.php index 5e16599..449fdb7 100755 --- a/src/Engines/MySql/Import/Primary.php +++ b/src/Engines/MySql/Import/Primary.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/3/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines\MySql\Import; @@ -34,21 +19,14 @@ class Primary extends Command implements CommandInterface { - /** @var LocalConnectionLoader */ - private $localConnection; - - /** @var array */ - private $properties; - - /** @var LoggerInterface */ - private $logger; - - /** @var Configuration */ - private $configuration; - - /** @var ConsoleOutput */ - private $output; - + private LocalConnectionLoader $localConnection; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; + private LoggerInterface $logger; + private Configuration $configuration; + private ConsoleOutput $output; + + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( LocalConnectionLoader $localConnection, Configuration $configuration, @@ -64,7 +42,7 @@ public function __construct( return parent::__construct('import-data-from-system-primary'); } - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { $transport->getLogger()->notice("Import database from var/ into local MySQL started"); $this->output->writeln("Import database from var/ into local MySQL started"); @@ -89,8 +67,7 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm } $this->output->writeln("Creating Local Database: " . - $this->getDatabaseCommand($environment) - ); + $this->getDatabaseCommand($environment)); if (!$conn->query($this->getDatabaseCommand($environment))) { $this->output->writeln('Database cannot be created: ' . $conn->errorInfo()[2] . ''); @@ -104,11 +81,10 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm ) ); $this->output->writeln("Local connection string: " . str_replace( - $this->localConnection->getPassword(), - '', - $this->assembleCommand(Download::DOWNLOAD_PATH_KEY) - ) - ); + $this->localConnection->getPassword(), + '', + $this->assembleCommand(Download::DOWNLOAD_PATH_KEY) + )); $preserved = $this->preserve(); @@ -127,27 +103,29 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm ); return $transport->withStatus(new Status('db_import', 'success')); } - } - public function getDatabaseCommand(EnvironmentInterface $environment) + public function getDatabaseCommand(EnvironmentInterface $environment): string { - return "CREATE DATABASE {$this->localConnection->getDatabase()}"; + return "CREATE DATABASE IF NOT EXISTS {$this->localConnection->getDatabase()}"; } - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - public function assembleCommand(string $filename) + public function assembleCommand(string $filename): string { return implode(' ', $this->getImportCommand($filename)); } - private function getImportCommand(string $filename) + /** + * @return string[] + */ + private function getImportCommand(string $filename): array { - $date = date('Y-m-d'); return [ "mysql -u \"{$this->localConnection->getUser()}\"", "-h {$this->localConnection->getHost()}", @@ -158,6 +136,7 @@ private function getImportCommand(string $filename) ]; } + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification private function preserve(): array { $connection = $this->getConnection(); @@ -198,7 +177,10 @@ private function preserve(): array return $output; } - private function getColumns($tableName): array + /** + * @return string[] + */ + private function getColumns(string $tableName): array { $connection = $this->getConnection(); $columns = []; @@ -209,8 +191,10 @@ private function getColumns($tableName): array } while ($row = $statement->fetch(PDO::FETCH_ASSOC)) { - if (isset($row['Extra']) - && $row['Extra'] === 'auto_increment') { + if ( + isset($row['Extra']) + && $row['Extra'] === 'auto_increment' + ) { continue; } @@ -220,11 +204,15 @@ private function getColumns($tableName): array return $columns; } + /** + * @param string[] $columnNames + */ private function flattenColumns(array $columnNames): string { return implode(', ', $columnNames); } + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint private function restore(array $values): void { foreach ($values as $tableName => $rows) { @@ -234,12 +222,12 @@ private function restore(array $values): void $tableColumnNames = $this->getColumns($tableName); $columnNames = $this->flattenColumns($tableColumnNames); $columnFillers = implode(', ', array_fill(0, count($row), '?')); - $valuesList = implode(', ', array_map(function($key) { + $valuesList = implode(', ', array_map(function ($key) { return "`${key}` = VALUES(`${key}`)"; }, array_keys($row))); $statement = $connection->prepare("INSERT INTO ${tableName} (${columnNames}) VALUES(${columnFillers})" - ." ON DUPLICATE KEY UPDATE ${valuesList}"); + . " ON DUPLICATE KEY UPDATE ${valuesList}"); $statement->execute(array_values($row)); } } diff --git a/src/Engines/MySql/Sandbox/Connection.php b/src/Engines/MySql/Sandbox/Connection.php index 22f4250..f32d33d 100755 --- a/src/Engines/MySql/Sandbox/Connection.php +++ b/src/Engines/MySql/Sandbox/Connection.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/25/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines\MySql\Sandbox; @@ -43,14 +28,12 @@ public function useSsl(): bool return true; } - public function test($onFailure): void + public function test(callable $onFailure): void { try { $this->getConnection(); } catch (\Exception $ex) { - if (is_callable($onFailure)) { - $onFailure($this); - } + $onFailure($this); } } @@ -80,7 +63,8 @@ public function getCharset(): string public function getDSN(): string { - return "mysql:host={$this->getHost()};dbname={$this->getDatabase()};port={$this->getPort()};charset={$this->getCharset()}"; + return "mysql:host={$this->getHost()};dbname={$this->getDatabase()};" + . "port={$this->getPort()};charset={$this->getCharset()}"; } public function getHost(): string @@ -108,6 +92,7 @@ public function getPassword(): string return $this->sandbox->getPassword(); } + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification public function getPreserve(): array { return []; diff --git a/src/Engines/MySql/Sandbox/Export.php b/src/Engines/MySql/Sandbox/Export.php index 582ce46..af99c1f 100755 --- a/src/Engines/MySql/Sandbox/Export.php +++ b/src/Engines/MySql/Sandbox/Export.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/3/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines\MySql\Sandbox; @@ -27,7 +12,6 @@ use Driver\Pipeline\Transport\TransportInterface; use Driver\System\Configuration; use Driver\System\Random; -use Driver\System\Tag; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Output\ConsoleOutput; @@ -38,13 +22,14 @@ class Export extends Command implements CommandInterface, CleanupInterface private Random $random; private ?string $filename = null; private Configuration $configuration; - private array $properties = []; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; private Utilities $utilities; private ConsoleOutput $output; - private Tag $tag; - - private $files = []; + /** @var string[] */ + private array $files = []; + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( RemoteConnectionInterface $connection, Ssl $ssl, @@ -65,19 +50,22 @@ public function __construct( return parent::__construct('mysql-sandbox-export'); } - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { - $this->connection->test(function(Connection $connection) { + $this->connection->test(function (Connection $connection): void { $connection->authorizeIp(); }); $transport->getLogger()->notice("Exporting database from remote MySql RDS"); - $this->output->writeln("Exporting database from remote MySql RDS. Please wait... this will take a long time."); + $this->output->writeln( + "Exporting database from remote MySql RDS. Please wait... this will take a long time." + ); $environmentName = $environment->getName(); $command = $this->assembleCommand($environmentName, $environment->getIgnoredTables()); @@ -97,16 +85,20 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm } } - public function cleanup(TransportInterface $transport, EnvironmentInterface $environment) + public function cleanup(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { - array_walk($this->files, function($fileName) { + array_walk($this->files, function ($fileName): void { if ($fileName && file_exists($fileName)) { @unlink($fileName); } }); + return $transport; } - private function assembleCommand($environmentName, $ignoredTables) + /** + * @param string[] $ignoredTables + */ + private function assembleCommand(string $environmentName, array $ignoredTables): string { $command = implode(' ', array_merge([ "mysqldump --user={$this->connection->getUser()}", @@ -133,27 +125,27 @@ private function assembleCommand($environmentName, $ignoredTables) return $command; } - private function getIgnoredTables($ignoredTables) + /** + * @param string[] $ignoredTables + * @return string[] + */ + private function getIgnoredTables(array $ignoredTables): array { - if (!is_array($ignoredTables)) { - $ignoredTables = []; - } - - $tableNames = array_filter(array_map(function($oldTableName) { + $tableNames = array_filter(array_map(function ($oldTableName) { return $this->utilities->tableName($oldTableName); }, $ignoredTables)); - return array_map(function($tableName) { - return "--ignore-table=".$this->connection->getDatabase().".".$tableName; + return array_map(function ($tableName) { + return "--ignore-table=" . $this->connection->getDatabase() . "." . $tableName; }, $tableNames); } - private function compressOutput() + private function compressOutput(): bool { return (bool)$this->configuration->getNode('configuration/compress-output') === true; } - private function getFilename($environmentName) + private function getFilename(string $environmentName): string { if ($this->filename) { return $this->filename; @@ -161,10 +153,11 @@ private function getFilename($environmentName) $path = $this->configuration->getNode('connections/rds/dump-path'); if (!$path) { - $path ='/tmp'; + $path = '/tmp'; } do { - $file = $path . '/driver_tmp_' . $environmentName . '_' . $this->random->getRandomString(10) . ($this->compressOutput() ? '.gz' : '.sql'); + $file = $path . '/driver_tmp_' . $environmentName . '_' . $this->random->getRandomString(10) + . ($this->compressOutput() ? '.gz' : '.sql'); } while (file_exists($file)); $this->filename = $file; diff --git a/src/Engines/MySql/Sandbox/Import.php b/src/Engines/MySql/Sandbox/Import.php index ee8fec3..771489a 100755 --- a/src/Engines/MySql/Sandbox/Import.php +++ b/src/Engines/MySql/Sandbox/Import.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/3/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines\MySql\Sandbox; @@ -24,7 +9,6 @@ use Driver\Pipeline\Environment\EnvironmentInterface; use Driver\Pipeline\Transport\Status; use Driver\Pipeline\Transport\TransportInterface; -use Driver\System\DebugMode; use Driver\System\LocalConnectionLoader; use Driver\System\Logs\LoggerInterface; use Symfony\Component\Console\Command\Command; @@ -35,18 +19,18 @@ class Import extends Command implements CommandInterface private LocalConnectionLoader $localConnection; private RemoteConnectionInterface $remoteConnection; private Ssl $ssl; - private array $properties = []; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; private LoggerInterface $logger; private ConsoleOutput $output; - private DebugMode $debugMode; + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( LocalConnectionLoader $localConnection, Ssl $ssl, RemoteConnectionInterface $connection, LoggerInterface $logger, ConsoleOutput $output, - DebugMode $debugMode, array $properties = [] ) { $this->localConnection = $localConnection; @@ -58,19 +42,22 @@ public function __construct( return parent::__construct('mysql-sandbox-import'); } - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { - $this->remoteConnection->test(function(RemoteConnectionInterface $connection) { + $this->remoteConnection->test(function (RemoteConnectionInterface $connection): void { $connection->authorizeIp(); }); - $this->output->writeln("Importing database into RDS. Please wait... This will take a long time."); + $this->output->writeln( + "Importing database into RDS. Please wait... This will take a long time." + ); $this->logger->notice("Importing database into RDS"); - $results = system($this->assembleCommand($transport->getData('dump-file'))); + $resultCode = 0; + system($this->assembleCommand($transport->getData('dump-file')), $resultCode); - if ($results) { - throw new \Exception('Import to RDS instance failed: ' . $results); - $this->output->writeln('Import to RDS instance failed: ' . $results . ''); + if ($resultCode !== 0) { + $this->output->writeln('Import to RDS instance failed.'); + throw new \Exception('Import to RDS instance failed.'); } else { $this->logger->notice("Import to RDS completed."); $this->output->writeln('Import to RDS completed.'); @@ -78,12 +65,13 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm } } - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - public function assembleCommand($path) + public function assembleCommand(string $path): string { $command = implode(' ', [ "mysql --user={$this->remoteConnection->getUser()}", @@ -96,7 +84,12 @@ public function assembleCommand($path) $path ]); - if (stripos($this->localConnection->getConnection()->getAttribute(\PDO::ATTR_SERVER_VERSION), 'maria') !== false) { + if ( + stripos( + $this->localConnection->getConnection()->getAttribute(\PDO::ATTR_SERVER_VERSION), + 'maria' + ) !== false + ) { $command = str_replace('--ssl-mode=VERIFY_CA', '--ssl', $command); } diff --git a/src/Engines/MySql/Sandbox/Init.php b/src/Engines/MySql/Sandbox/Init.php index 0de80d6..02b284a 100755 --- a/src/Engines/MySql/Sandbox/Init.php +++ b/src/Engines/MySql/Sandbox/Init.php @@ -1,22 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/19/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines\MySql\Sandbox; @@ -34,16 +18,18 @@ class Init extends Command implements CommandInterface { private Configuration $configuration; private Sandbox $sandbox; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification private array $properties = []; private ConsoleOutput $output; private DebugMode $debugMode; + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( Configuration $configuration, Sandbox $sandbox, ConsoleOutput $output, DebugMode $debugMode, - $properties = [] + array $properties = [] ) { $this->configuration = $configuration; $this->sandbox = $sandbox; @@ -54,12 +40,13 @@ public function __construct( return parent::__construct('mysql-sandbox-init'); } - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { if ($this->debugMode->get()) { $transport->getLogger()->notice('No sandbox initialization due to debug mode enabled.'); @@ -85,7 +72,7 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm $progressBar = new ProgressBar($this->output, $maxTries); $progressBar->setFormat('minimal'); - while(!($active = $this->sandbox->getInstanceActive()) && $tries < $maxTries) { + while (!($active = $this->sandbox->getInstanceActive()) && $tries < $maxTries) { $transport->getLogger()->notice('Checking if sandbox is active.'); $tries++; sleep(10); @@ -107,7 +94,7 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm return $transport->withStatus(new Status('sandbox_init', 'success')); } - public function signalHandler($signal) + public function signalHandler(int $signal): void { if ($signal !== SIGINT && $signal !== SIGKILL) { return; @@ -121,5 +108,4 @@ public function signalHandler($signal) $this->output->write('Failed to shut down RDS instance. Please login to AWS and kill the instance.'); } } - } diff --git a/src/Engines/MySql/Sandbox/Sandbox.php b/src/Engines/MySql/Sandbox/Sandbox.php index 3c83804..c30de1f 100755 --- a/src/Engines/MySql/Sandbox/Sandbox.php +++ b/src/Engines/MySql/Sandbox/Sandbox.php @@ -1,24 +1,12 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/19/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines\MySql\Sandbox; +use Aws\Ec2\Ec2Client; +use Aws\Rds\RdsClient; +use Aws\Result; use Driver\System\AwsClientFactory; use Driver\System\Configuration; use Driver\System\Logs\LoggerInterface; @@ -29,26 +17,25 @@ class Sandbox { - const DEFAULT_ENGINE = 'MySQL'; - - private $configuration; - private $instance; - private $initialized; - private $remoteIpFetcher; - private $logger; - private $random; - private $awsClientFactory; - private $securityGroupId; - private $securityGroupName; - private $dbName; - private $identifier; - private $username; - private $password; - private $statuses; - private $output; - - /** @var \Symfony\Component\EventDispatcher\EventDispatcher */ + private const DEFAULT_ENGINE = 'MySQL'; + + private Configuration $configuration; + private RemoteIP $remoteIpFetcher; + private LoggerInterface $logger; + private Random $random; + private AwsClientFactory $awsClientFactory; + private ConsoleOutput $output; private EventDispatcher $eventDispatcher; + private ?Result $instance = null; + private bool $initialized = false; + private ?string $securityGroupId = null; + private ?string $securityGroupName = null; + private ?string $dbName = null; + private ?string $identifier = null; + private ?string $username = null; + private ?string $password = null; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $statuses = []; public function __construct( Configuration $configuration, @@ -58,7 +45,7 @@ public function __construct( AwsClientFactory $awsClientFactory, ConsoleOutput $output, EventDispatcher $eventDispatcher, - $disableInstantiation = true + bool $disableInstantiation = true ) { $this->configuration = $configuration; $this->remoteIpFetcher = $remoteIpFetcher; @@ -72,12 +59,16 @@ public function __construct( $this->eventDispatcher = $eventDispatcher; } - public function init() + public function init(): bool { $this->logger->info("Using RDS instance: " . $this->getIdentifier()); $this->output->writeln("Using RDS instance: " . $this->getIdentifier() . ''); - if ($this->initialized || $this->configuration->getNode('connections/rds/instance-name') || $this->getInstanceActive()) { + if ( + $this->initialized + || $this->configuration->getNode('connections/rds/instance-name') + || $this->getInstanceActive() + ) { $this->logger->info("Using RDS instance: " . $this->getIdentifier()); $this->output->writeln("Using RDS instance: " . $this->getIdentifier() . ''); return false; @@ -129,11 +120,13 @@ public function init() } } - public function shutdown() + public function shutdown(): bool { if ($this->configuration->getNode('connections/rds/instance-name')) { $this->logger->info("Using static RDS instance and will not shutdown: " . $this->getIdentifier()); - $this->output->writeln("Using static RDS instance and will not shutdown: " . $this->getIdentifier() . ''); + $this->output->writeln( + "Using static RDS instance and will not shutdown: " . $this->getIdentifier() . '' + ); return false; } @@ -151,7 +144,10 @@ public function shutdown() return true; } - public function getJson() + /** + * @return array + */ + public function getJson(): array { return [ 'host' => $this->getEndpointAddress(), @@ -162,35 +158,37 @@ public function getJson() ]; } - public function getInstanceActive() + public function getInstanceActive(): bool { $status = $this->getInstanceStatus(); - return isset($status['DBInstanceStatus']) && ($status['DBInstanceStatus'] === "available" || $status['DBInstanceStatus'] === "backing_up"); + return isset($status['DBInstanceStatus']) + && ($status['DBInstanceStatus'] === "available" || $status['DBInstanceStatus'] === "backing_up"); } - public function getEndpointAddress() + public function getEndpointAddress(): ?string { $status = $this->getInstanceStatus(); - return isset($status['Endpoint']['Address']) ? $status['Endpoint']['Address'] : null; + return $status['Endpoint']['Address'] ?? null; } - public function getEndpointPort() + public function getEndpointPort(): string { $status = $this->getInstanceStatus(); - return isset($status['Endpoint']['Port']) ? $status['Endpoint']['Port'] : 3306; + return (string)($status['Endpoint']['Port'] ?? 3306); } - public function getDbParameterGroupName() + public function getDbParameterGroupName(): string { $value = $this->configuration->getNode('connections/rds/parameter-group-name'); if (!is_array($value) && $value) { return $value; } else { - return false; + return ''; } } + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint public function getInstanceStatus() { try { @@ -206,25 +204,7 @@ public function getInstanceStatus() } } - private function getSecurityGroup() - { - if (!$this->securityGroupId) { - $client = $this->getEc2Client(); - - $securityGroup = $client->createSecurityGroup([ - 'GroupName' => $this->getSecurityGroupName(), - 'Description' => 'Temporary security group for Driver uploads' - ]); - - $this->authorizeIp(); - - $this->securityGroupId = $securityGroup['GroupId']; - } - - return $this->securityGroupId; - } - - public function authorizeIp() + public function authorizeIp(): void { try { $this->getEc2Client()->authorizeSecurityGroupIngress([ @@ -244,21 +224,16 @@ public function authorizeIp() ]); } catch (\Exception $ex) { if (stripos($ex->getMessage(), 'InvalidPermission.Duplicate') === false) { - throw $ex; $this->output->writeln("Exception: " . $ex->getMessage()); + throw $ex; } } } - private function getPublicIp() - { - return $this->remoteIpFetcher->getPublicIP(); - } - - public function getDBName() + public function getDBName(): string { if (!$this->dbName) { - $this->dbName = $this->configuration->getNode('connections/rds/instance-db-name'); + $this->dbName = (string)$this->configuration->getNode('connections/rds/instance-db-name'); if (!$this->dbName) { $this->dbName = 'd' . $this->getRandomString(12); @@ -268,106 +243,119 @@ public function getDBName() return $this->dbName; } - private function getIdentifier() + public function getSecurityGroupName(): string { - if (!$this->identifier) { - $this->identifier = $this->configuration->getNode('connections/rds/instance-identifier'); + if (!$this->securityGroupName) { + $this->securityGroupName = (string)$this->configuration->getNode('connections/rds/security-group-name'); - if (!$this->identifier) { - $this->identifier = 'driver-upload-' . $this->getRandomString(6); + if (!$this->securityGroupName) { + $this->securityGroupName = 'driver-temp-' . $this->getRandomString(6); } } - return $this->identifier; + return $this->securityGroupName; } - private function getStorageType() + public function getUsername(): string { - $storageType = $this->configuration->getNode('connections/rds/storage-type'); + if (!$this->username) { + $this->username = (string)$this->configuration->getNode('connections/rds/instance-username'); - if (!$storageType) { - $storageType = 'standard'; + if (!$this->username) { + $this->username = 'u' . $this->getRandomString(12); + } } - return $storageType; + return $this->username; } - private function getEngine(): string + public function getPassword(): string { - return $this->configuration->getNode('connections/rds/engine') ?? self::DEFAULT_ENGINE; - } + if (!$this->password) { + $this->password = (string)$this->configuration->getNode('connections/rds/instance-password'); - private function getEngineVersion(): ?string - { - return $this->configuration->getNode('connections/rds/engine-version'); + if (!$this->password) { + $this->password = $this->getRandomString(30); + } + } + + return $this->password; } - public function getSecurityGroupName() + private function getSecurityGroup(): string { - if (!$this->securityGroupName) { - $this->securityGroupName = $this->configuration->getNode('connections/rds/security-group-name'); + if (!$this->securityGroupId) { + $client = $this->getEc2Client(); - if (!$this->securityGroupName) { - $this->securityGroupName = 'driver-temp-' . $this->getRandomString(6); - } + $securityGroup = $client->createSecurityGroup([ + 'GroupName' => $this->getSecurityGroupName(), + 'Description' => 'Temporary security group for Driver uploads' + ]); + + $this->authorizeIp(); + + $this->securityGroupId = $securityGroup['GroupId']; } - return $this->securityGroupName; + return $this->securityGroupId; } - public function getUsername() + private function getPublicIp(): string { - if (!$this->username) { - $this->username = $this->configuration->getNode('connections/rds/instance-username'); + return $this->remoteIpFetcher->getPublicIP(); + } - if (!$this->username) { - $this->username = 'u' . $this->getRandomString(12); + private function getIdentifier(): string + { + if (!$this->identifier) { + $this->identifier = (string)$this->configuration->getNode('connections/rds/instance-identifier'); + + if (!$this->identifier) { + $this->identifier = 'driver-upload-' . $this->getRandomString(6); } } - return $this->username; + return $this->identifier; } - public function getPassword() + private function getStorageType(): string { - if (!$this->password) { - $this->password = $this->configuration->getNode('connections/rds/instance-password'); + $storageType = $this->configuration->getNode('connections/rds/storage-type'); - if (!$this->password) { - $this->password = $this->getRandomString(30); - } + if (!$storageType) { + $storageType = 'standard'; } - return $this->password; + return $storageType; } - private function getRandomString($length) + private function getEngine(): string + { + return (string)($this->configuration->getNode('connections/rds/engine') ?? self::DEFAULT_ENGINE); + } + + private function getEngineVersion(): ?string + { + return $this->configuration->getNode('connections/rds/engine-version'); + } + + private function getRandomString(int $length): string { return $this->random->getRandomString($length); } - /** - * @return \Aws\Ec2\Ec2Client - */ - private function getEc2Client() + private function getEc2Client(): Ec2Client { return $this->awsClientFactory->create('Ec2', $this->getAwsParameters("ec2", '2016-09-15')); } - /** - * @return \Aws\Rds\RdsClient - */ - private function getRdsClient() + private function getRdsClient(): RdsClient { return $this->awsClientFactory->create('Rds', $this->getAwsParameters("rds", '2014-10-31')); } - /** - * @param $type - * @param $version - * @return array - */ - private function getAwsParameters($type, $version) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + private function getAwsParameters(string $type, string $version): array { $parameters = [ 'credentials' => [ @@ -383,7 +371,9 @@ private function getAwsParameters($type, $version) ]; if (empty($parameters['region'])) { - $this->output->writeln('No region specified. Are you sure that .driver/connections.yaml exists?'); + $this->output->writeln( + 'No region specified. Are you sure that .driver/connections.yaml exists?' + ); } return $parameters; diff --git a/src/Engines/MySql/Sandbox/Shutdown.php b/src/Engines/MySql/Sandbox/Shutdown.php index 56a6948..0958474 100755 --- a/src/Engines/MySql/Sandbox/Shutdown.php +++ b/src/Engines/MySql/Sandbox/Shutdown.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/19/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines\MySql\Sandbox; @@ -34,10 +19,12 @@ class Shutdown extends Command implements CommandInterface, ErrorInterface { private Configuration $configuration; private Sandbox $sandbox; - private array $properties = []; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; private ConsoleOutput $output; private DebugMode $debugMode; + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( Configuration $configuration, Sandbox $sandbox, @@ -54,26 +41,30 @@ public function __construct( return parent::__construct('mysql-sandbox-shutdown'); } - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { if ($this->debugMode->get()) { $transport->getLogger()->notice('No sandbox shutdown due to debug mode enabled.'); + return $transport->withStatus( + new Status('sandbox_shutdown', 'No sandbox shutdown due to debug mode enabled.') + ); } return $this->apply($transport, $environment); } - public function error(TransportInterface $transport, EnvironmentInterface $environment) + public function error(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { return $this->apply($transport, $environment); } - private function apply(TransportInterface $transport, EnvironmentInterface $environment) + private function apply(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { $transport->getLogger()->notice("Shutting down RDS"); $this->output->writeln("Shutting down RDS"); diff --git a/src/Engines/MySql/Sandbox/Ssl.php b/src/Engines/MySql/Sandbox/Ssl.php index d697ddb..91f1599 100755 --- a/src/Engines/MySql/Sandbox/Ssl.php +++ b/src/Engines/MySql/Sandbox/Ssl.php @@ -1,53 +1,24 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/25/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines\MySql\Sandbox; class Ssl { - const RDS_CA_URL = 'https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem'; - const SYSTEM_PATH = '/tmp/rds-combined-ca-bundle.pem'; + private const RDS_CA_URL = 'https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem'; + private const SYSTEM_PATH = '/tmp/rds-combined-ca-bundle.pem'; - public function getPath() + public function getPath(): ?string { if (!file_exists(self::SYSTEM_PATH)) { - $this->downloadTo(self::SYSTEM_PATH); + file_put_contents(self::SYSTEM_PATH, fopen(self::RDS_CA_URL, 'r')); } if (file_exists(self::SYSTEM_PATH)) { return self::SYSTEM_PATH; } else { - return false; + return null; } } - - public function mergeOptions($input) - { - if ($path = $this->getPath()) { - return array_merge($input, [ \PDO::MYSQL_ATTR_SSL_CA => $this->getPath() ]); - } else { - return $input; - } - } - - private function downloadTo($path) - { - file_put_contents($path, fopen(self::RDS_CA_URL, 'r')); - } -} \ No newline at end of file +} diff --git a/src/Engines/MySql/Sandbox/Utilities.php b/src/Engines/MySql/Sandbox/Utilities.php index 2975207..778724e 100755 --- a/src/Engines/MySql/Sandbox/Utilities.php +++ b/src/Engines/MySql/Sandbox/Utilities.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines\MySql\Sandbox; @@ -23,15 +8,16 @@ class Utilities { - private $connection; - private $cachedTables = false; + private LocalConnectionLoader $connection; + /** @var string[]|null */ + private ?array $cachedTables = null; public function __construct(LocalConnectionLoader $connection) { $this->connection = $connection; } - public function tableExists($tableName) + public function tableExists(string $tableName): bool { try { $result = $this->connection->getConnection()->query("SELECT 1 FROM $tableName LIMIT 1"); @@ -42,28 +28,9 @@ public function tableExists($tableName) return $result !== false; } -// public function clearTable($tableName) -// { -// $connection = $this->connection->getConnection(); -// try { -// $connection->beginTransaction(); -// -// if ($this->tableExists($tableName)) { -// $connection->query("set foreign_key_checks=0"); -// $connection->query("TRUNCATE TABLE {$tableName}"); -// } -// -// $connection->commit(); -// } catch (\Exception $ex) { -// $connection->rollBack(); -// } finally { -// $connection->query("set foreign_key_checks=1"); -// } -// } - - public function tableName($tableName) + public function tableName(string $tableName): string { - $fullTableName = array_reduce($this->getTables(), function ($carry, $sourceTableName) use ($tableName) { + return array_reduce($this->getTables(), function ($carry, $sourceTableName) use ($tableName) { if (strlen($sourceTableName) < strlen($tableName)) { return $carry; } @@ -72,24 +39,31 @@ public function tableName($tableName) return $tableName; } - if (substr_compare($sourceTableName, $tableName, strlen($sourceTableName) - strlen($tableName), strlen($tableName)) === 0) { + if ( + substr_compare( + $sourceTableName, + $tableName, + strlen($sourceTableName) - strlen($tableName), + strlen($tableName) + ) === 0 + ) { return $sourceTableName; } return $carry; }, ''); - - return $fullTableName; } - private function getTables() + /** + * @return array|string[] + */ + private function getTables(): array { - if ($this->cachedTables === false) { + if ($this->cachedTables === null) { $result = $this->connection->getConnection()->query('SHOW TABLES;'); - - $this->cachedTables = $result->fetchAll(\PDO::FETCH_COLUMN, 0); + $this->cachedTables = $result->fetchAll(\PDO::FETCH_COLUMN, 0) ?: []; } return $this->cachedTables; } -} \ No newline at end of file +} diff --git a/src/Engines/MySql/Transformation.php b/src/Engines/MySql/Transformation.php index 6d84201..19dab2d 100644 --- a/src/Engines/MySql/Transformation.php +++ b/src/Engines/MySql/Transformation.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/17/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines\MySql; @@ -34,15 +19,16 @@ class Transformation extends Command implements CommandInterface { private Configuration $configuration; - private array $properties = []; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; private RemoteConnectionInterface $connection; private LoggerInterface $logger; private ConsoleOutput $output; + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( Configuration $configuration, RemoteConnectionInterface $connection, - Utilities $utilities, LoggerInterface $logger, ConsoleOutput $output, array $properties = [] @@ -56,10 +42,12 @@ public function __construct( parent::__construct('mysql-transformation'); } - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { - if (count($environment->getOnlyForPipeline()) - && !in_array($transport->getPipeline(), $environment->getOnlyForPipeline())) { + if ( + count($environment->getOnlyForPipeline()) + && !in_array($transport->getPipeline(), $environment->getOnlyForPipeline()) + ) { return $transport->withStatus(new Status('mysql-transformation', 'stale')); } @@ -67,14 +55,18 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm return $transport->withStatus(new Status('mysql-transformation', 'success')); } - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - private function applyTransformationsTo(ReconnectingPDO $connection, $transformations) + /** + * @param string[] $transformations + */ + private function applyTransformationsTo(ReconnectingPDO $connection, array $transformations): void { - array_walk($transformations, function ($query) use ($connection) { + array_walk($transformations, function ($query) use ($connection): void { try { $this->logger->info("Attempting: " . $query); $this->output->writeln(" Attempting: " . $query . ''); diff --git a/src/Engines/MySql/Transformation/Anonymize.php b/src/Engines/MySql/Transformation/Anonymize.php index e84ab99..bd5aafd 100644 --- a/src/Engines/MySql/Transformation/Anonymize.php +++ b/src/Engines/MySql/Transformation/Anonymize.php @@ -21,10 +21,12 @@ class Anonymize extends Command implements CommandInterface { private Configuration $configuration; private RemoteConnectionInterface $connection; - private array $properties = []; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; private Seed $seed; private ConsoleOutput $output; + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( Configuration $configuration, RemoteConnectionInterface $connection, @@ -44,7 +46,8 @@ public function __construct( public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { $config = $this->configuration->getNode('anonymize'); - if ((isset($config['disabled']) && $config['disabled'] === true) + if ( + (isset($config['disabled']) && $config['disabled'] === true) || !isset($config['tables']) || !isset($config['seed']) ) { @@ -68,11 +71,15 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm return $transport->withStatus(new Status('mysql-transformation-anonymize', 'success')); } + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification public function getProperties(): array { return $this->properties; } + /** + * @param array> $columns + */ private function anonymize(string $table, array $columns): void { $connection = $this->connection->getConnection(); @@ -82,7 +89,9 @@ private function anonymize(string $table, array $columns): void $connection->query("SET foreign_key_checks = 0;"); $connection->query("TRUNCATE ${table};"); $connection->query("SET foreign_key_checks = 1;"); - } catch (\Exception $ex) {} + } catch (\Exception $ex) { + // Do nothing + } } foreach ($columns as $columnName => $description) { @@ -132,6 +141,9 @@ private function queryEmpty(): string return '""'; } + /** + * @param array $description + */ private function getTypeMethod(array $description): string { $type = $description['type'] ?? 'general'; diff --git a/src/Engines/MySql/Transformation/Anonymize/Seed.php b/src/Engines/MySql/Transformation/Anonymize/Seed.php index cfecc98..328c025 100644 --- a/src/Engines/MySql/Transformation/Anonymize/Seed.php +++ b/src/Engines/MySql/Transformation/Anonymize/Seed.php @@ -1,9 +1,6 @@ clean(); } @@ -65,13 +62,13 @@ public function getCount(): int return $this->count; } - private function clean() + private function clean(): void { $this->connection->getConnection()->query('DROP TABLE IF EXISTS ' . self::FAKE_USER_TABLE); $this->count = 0; } - private function createTable() + private function createTable(): void { $table = self::FAKE_USER_TABLE; diff --git a/src/Engines/MySql/Transformation/Reduce.php b/src/Engines/MySql/Transformation/Reduce.php index 875b8af..eadfb7c 100644 --- a/src/Engines/MySql/Transformation/Reduce.php +++ b/src/Engines/MySql/Transformation/Reduce.php @@ -1,9 +1,6 @@ configuration->getNode('reduce/tables'); @@ -67,7 +66,9 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm } catch (\Exception $ex) { $this->logger->error('An error occurred when running this query: ' . $query); $this->logger->error($ex->getMessage()); - $this->output->writeln('An error occurred when running this query: ' . $query. $ex->getMessage() . ''); + $this->output->writeln( + 'An error occurred when running this query: ' . $query . $ex->getMessage() . '' + ); } } @@ -77,9 +78,9 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm return $transport->withStatus(new Status('mysql-transformation-reduce', 'success')); } - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - } diff --git a/src/Engines/MySql/Transformation/UpdateValues.php b/src/Engines/MySql/Transformation/UpdateValues.php index 3822134..cf098ae 100644 --- a/src/Engines/MySql/Transformation/UpdateValues.php +++ b/src/Engines/MySql/Transformation/UpdateValues.php @@ -23,14 +23,16 @@ class UpdateValues extends Command implements CommandInterface { - const COMMAND_NAME = 'mysql-transformation-update-values'; + private const COMMAND_NAME = 'mysql-transformation-update-values'; private Configuration $configuration; private RemoteConnectionInterface $connection; private ConsoleOutput $output; private QueryBuilder $queryBuilder; - private array $properties = []; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( Configuration $configuration, RemoteConnectionInterface $connection, @@ -66,11 +68,15 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm return $transport->withStatus(new Status(self::COMMAND_NAME, 'success')); } + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification public function getProperties(): array { return $this->properties; } + /** + * @param array> $data + */ private function update(string $table, array $data): void { $joins = $this->buildJoins($data); @@ -88,6 +94,7 @@ private function update(string $table, array $data): void } /** + * @param array> $data * @return Join[] */ private function buildJoins(array $data): array @@ -115,6 +122,7 @@ private function buildJoins(array $data): array } /** + * @param array> $data * @return Value[] */ private function buildValues(array $data): array diff --git a/src/Engines/ReconnectingPDO.php b/src/Engines/ReconnectingPDO.php index b8bc2fb..7d8a593 100644 --- a/src/Engines/ReconnectingPDO.php +++ b/src/Engines/ReconnectingPDO.php @@ -8,6 +8,7 @@ use PDOException; use PDOStatement; +// phpcs:disable Generic.Files.LineLength /** * @method PDOStatement|false prepare($query, array $options = []) * @method bool beginTransaction() @@ -34,6 +35,7 @@ * @method array|false pgsqlGetNotify(int $fetchMode = 0, int $timeoutMilliseconds = 0) * @method int pgsqlGetPid() */ +// phpcs:enable class ReconnectingPDO { private const MYSQL_GENERAL_ERROR_CODE = 'HY000'; @@ -42,6 +44,7 @@ class ReconnectingPDO private string $dsn; private ?string $username; private ?string $password; + /** @var array|null */ private ?array $options; private PDO $pdo; @@ -60,12 +63,14 @@ public function __construct(string $dsn, ?string $username = null, ?string $pass /** * @throws PDOException */ + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification,SlevomatCodingStandard.TypeHints.ReturnTypeHint public function __call(string $name, array $arguments) { try { $this->pdo->query('SELECT 1')->fetchColumn(); } catch (PDOException $e) { - if ($e->errorInfo[0] !== self::MYSQL_GENERAL_ERROR_CODE + if ( + $e->errorInfo[0] !== self::MYSQL_GENERAL_ERROR_CODE || $e->errorInfo[1] !== self::SERVER_HAS_GONE_AWAY_ERROR_CODE ) { throw $e; diff --git a/src/Engines/RemoteConnectionInterface.php b/src/Engines/RemoteConnectionInterface.php index c939b21..8099164 100644 --- a/src/Engines/RemoteConnectionInterface.php +++ b/src/Engines/RemoteConnectionInterface.php @@ -1,27 +1,12 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/3/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines; interface RemoteConnectionInterface extends ConnectionInterface { public function useSsl(): bool; - public function test($onFailure): void; + public function test(callable $onFailure): void; public function authorizeIp(): void; } diff --git a/src/Engines/S3/Download.php b/src/Engines/S3/Download.php index a77fb0e..80582bc 100644 --- a/src/Engines/S3/Download.php +++ b/src/Engines/S3/Download.php @@ -1,24 +1,10 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/3/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines\S3; +use Aws\Result; use Aws\S3\S3Client; use Driver\Commands\CommandInterface; use Driver\Pipeline\Environment\EnvironmentInterface; @@ -34,16 +20,18 @@ class Download extends Command implements CommandInterface { - const DOWNLOAD_PATH_KEY = 'download_path'; + public const DOWNLOAD_PATH_KEY = 'download_path'; private LocalConnectionLoader $localConnection; private Configuration $configuration; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification private array $properties; private LoggerInterface $logger; private ConsoleOutput $output; private EnvironmentManager $environmentManager; private S3FilenameFormatter $s3FilenameFormatter; + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( LocalConnectionLoader $localConnection, Configuration $configuration, @@ -64,7 +52,7 @@ public function __construct( parent::__construct('s3-download'); } - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { try { $filename = $this->s3FilenameFormatter->execute($environment, $this->getFileKey()); @@ -74,7 +62,6 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm ); $this->output->writeln( sprintf("Beginning file download from: s3://%s/%s", $this->getBucket(), $filename) - ); $date = date('Y-m-d'); $client = $this->getS3Client(); @@ -93,7 +80,11 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm sprintf("Downloaded file from: s3://%s/%s", $this->getBucket(), $this->getFileName($environment)) ); $this->output->writeln( - sprintf("Downloaded file from: s3://%s/%s to project var/ directory", $this->getBucket(), $this->getFileName($environment)) + sprintf( + "Downloaded file from: s3://%s/%s to project var/ directory", + $this->getBucket(), + $this->getFileName($environment) + ) ); if (strpos($this->getFileName($environment), ".gz") !== false) { @@ -105,7 +96,7 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm ->withNewData(self::DOWNLOAD_PATH_KEY, $outputFile); } catch (\Exception $ex) { $this->output->section(); - $this->output->writeln('Failed getting object from S3: ' . $ex->getTraceAsString(). ''); + $this->output->writeln('Failed getting object from S3: ' . $ex->getTraceAsString() . ''); $this->logger->error('Failed getting object from S3: ' . $ex->getMessage(), [ $ex->getMessage(), $ex->getTraceAsString() @@ -115,22 +106,21 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm } } - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - protected function getObjectUrl(\Aws\Result $data) + private function getObjectUrl(Result $data): string { - return $data->get('ObjectURL'); + return (string)$data->get('ObjectURL'); } - private function getFileName(EnvironmentInterface $environment) + private function getFileName(EnvironmentInterface $environment): string { $replace = str_replace('{{environment}}', '-' . $environment->getName(), $this->getFileKey()); - $replace = str_replace('{{date}}', date('YmdHis'), $replace); - - return $replace; + return str_replace('{{date}}', date('YmdHis'), $replace); } private function getFileKey(): string @@ -142,7 +132,7 @@ private function getFileKey(): string } } - private function compressOutput() + private function compressOutput(): bool { return (bool)$this->configuration->getNode('configuration/compress-output') === true; } @@ -152,9 +142,9 @@ private function getBucket(): string return $this->configuration->getNodeString('connections/s3/bucket'); } - private function getDirectory() + private function getDirectory(): string { - $directory = $this->configuration->getNode('connections/s3/directory'); + $directory = (string)$this->configuration->getNode('connections/s3/directory'); if ($directory) { $directory .= '/'; } @@ -162,14 +152,15 @@ private function getDirectory() return $directory; } - private function getS3Client() + private function getS3Client(): S3Client { return new S3Client($this->getAwsParameters()); } - private function getAwsParameters() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + private function getAwsParameters(): array { - $parameters = [ + return [ 'credentials' => [ 'key' => $this->configuration->getNode("connections/s3/key") ?? $this->configuration->getNode("connections/aws/key"), @@ -180,6 +171,5 @@ private function getAwsParameters() ?? $this->configuration->getNode("connections/aws/region"), 'version' => '2006-03-01' ]; - return $parameters; } } diff --git a/src/Engines/S3/Upload.php b/src/Engines/S3/Upload.php index e221aa2..ed49607 100644 --- a/src/Engines/S3/Upload.php +++ b/src/Engines/S3/Upload.php @@ -1,24 +1,10 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/3/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Engines\S3; +use Aws\Result; use Aws\S3\S3Client; use Driver\Commands\CommandInterface; use Driver\Pipeline\Environment\EnvironmentInterface; @@ -35,14 +21,15 @@ class Upload extends Command implements CommandInterface { - protected Configuration $configuration; - protected array $properties; + private Configuration $configuration; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; private LoggerInterface $logger; private ConsoleOutput $output; private S3FilenameFormatter $s3FilenameFormatter; - /** @var \Symfony\Component\EventDispatcher\EventDispatcher */ private EventDispatcher $eventDispatcher; + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( Configuration $configuration, LoggerInterface $logger, @@ -61,9 +48,9 @@ public function __construct( $this->eventDispatcher = $eventDispatcher; } - public function go(TransportInterface $transport, EnvironmentInterface $environment) + public function go(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { - $this->eventDispatcher->addListener(ConsoleEvents::TERMINATE, function (ConsoleCommandEvent $event) { + $this->eventDispatcher->addListener(ConsoleEvents::TERMINATE, function (ConsoleCommandEvent $event): void { $this->output->writeln('Cancel registered!'); }); @@ -108,12 +95,13 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm } } - public function getProperties() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function getProperties(): array { return $this->properties; } - protected function getObjectUrl(\Aws\Result $data) + private function getObjectUrl(Result $data): string { return $data->get('ObjectURL'); } @@ -139,9 +127,9 @@ private function getBucket(): string return $this->configuration->getNodeString('connections/s3/bucket'); } - private function getDirectory() + private function getDirectory(): string { - $directory = $this->configuration->getNode('connections/s3/directory'); + $directory = (string)$this->configuration->getNode('connections/s3/directory'); if ($directory) { $directory .= '/'; } @@ -149,14 +137,15 @@ private function getDirectory() return $directory; } - private function getS3Client() + private function getS3Client(): S3Client { return new S3Client($this->getAwsParameters()); } - private function getAwsParameters() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + private function getAwsParameters(): array { - $parameters = [ + return [ 'credentials' => [ 'key' => $this->configuration->getNode("connections/s3/key") ?? $this->configuration->getNode("connections/aws/key"), @@ -167,6 +156,5 @@ private function getAwsParameters() ?? $this->configuration->getNode("connections/aws/region"), 'version' => '2006-03-01' ]; - return $parameters; } } diff --git a/src/Pipeline/Command.php b/src/Pipeline/Command.php index 6df182d..219a709 100644 --- a/src/Pipeline/Command.php +++ b/src/Pipeline/Command.php @@ -1,28 +1,10 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/28/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline; -use Driver\Commands\CommandInterface; -use Driver\Pipeline\Environment\EnvironmentInterface; use Driver\Pipeline\Environment\Manager as EnvironmentManager; -use Driver\Pipeline\Transport\TransportInterface; use Driver\System\Logs\LoggerInterface; use Driver\System\Tag; use Symfony\Component\Console\Command\Command as ConsoleCommand; @@ -34,17 +16,17 @@ use Symfony\Component\Console\Question\ChoiceQuestion; use Symfony\Component\Console\Question\Question; -class Command extends ConsoleCommand implements CommandInterface +class Command extends ConsoleCommand { - const PIPELINE = 'pipeline'; - const ENVIRONMENT = 'environment'; - const DEBUG = 'debug'; - const TAG = 'tag'; - const CI = 'ci'; + private const PIPELINE = 'pipeline'; + private const ENVIRONMENT = 'environment'; + private const DEBUG = 'debug'; + private const TAG = 'tag'; + private const CI = 'ci'; private Master $pipeMaster; - private EnvironmentManager $environmentManager; private LoggerInterface $logger; + private EnvironmentManager $environmentManager; private ConsoleOutput $output; private Tag $tag; @@ -64,19 +46,28 @@ public function __construct( parent::__construct(); } - protected function configure() + protected function configure(): void { $this->setName('run') - ->setDescription('This tool runs handles databases; importing and exporting. It is based on pipelines which are defined in YAML configuration. Specify which pipeline to run as an argument.'); + ->setDescription( + 'This tool runs handles databases; importing and exporting. ' + . 'It is based on pipelines which are defined in YAML configuration. ' + . 'Specify which pipeline to run as an argument.' + ); $this->addArgument(self::PIPELINE, InputArgument::OPTIONAL, 'The pipeline to execute.') - ->addOption(self::ENVIRONMENT, 'env', InputOption::VALUE_OPTIONAL, 'The environment(s) for which to run Driver.') + ->addOption( + self::ENVIRONMENT, + 'env', + InputOption::VALUE_OPTIONAL, + 'The environment(s) for which to run Driver.' + ) ->addOption(self::TAG, 'tag', InputOption::VALUE_OPTIONAL, 'A tag for the output file.') ->addOption(self::DEBUG, 'd', InputOption::VALUE_OPTIONAL, 'Enable debug mode') ->addOption(self::CI, 'ci', InputOption::VALUE_OPTIONAL, 'Run without asking questions'); } - protected function execute(InputInterface $input, OutputInterface $output) + protected function execute(InputInterface $input, OutputInterface $output): void { $output->writeln("Executing Pipeline Command..."); $this->logger->setParams($input, $output); @@ -84,7 +75,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $this->tag->setTag($input->getOption(self::TAG)); if ($pipeLine = $input->getArgument(self::PIPELINE)) { - $this->pipeMaster->run($pipeLine); + $transport = $this->pipeMaster->run($pipeLine); } else { if (!$input->hasOption(self::CI)) { $helper = $this->getHelper('question'); @@ -96,18 +87,21 @@ protected function execute(InputInterface $input, OutputInterface $output) $pipeLine = $helper->ask($input, $output, $pipelineQuestion); $this->askAboutEnv($input, $output); - } else { $pipeLine = Master::DEFAULT_NODE; } - $output->writeln('Running pipeline: '. $pipeLine . ''); + $output->writeln('Running pipeline: ' . $pipeLine . ''); + + $transport = $this->pipeMaster->run($pipeLine); + } - $this->pipeMaster->run($pipeLine); + foreach ($transport->getErrors() as $error) { + $output->writeln('' . $error->getNode() . ' - ' . $error->getMessage() . ''); } } - private function askAboutEnv($input, $output) + private function askAboutEnv(InputInterface $input, OutputInterface $output): void { $helper = $this->getHelper('question'); @@ -119,22 +113,10 @@ private function askAboutEnv($input, $output) ); $env = $helper->ask($input, $output, $envQuestion); - $output->writeln('Using: '. $env . ' environment'); + $output->writeln('Using: ' . $env . ' environment'); $input->setOption(self::ENVIRONMENT, $env); $this->environmentManager->setRunFor($env); } } - - - public function go(TransportInterface $transport, EnvironmentInterface $environment) - { - $this->output->writeln("The Pipe command cannot be included in a pipe. It is the mother of all pipes."); - throw new \Exception('The Pipe command cannot be included in a pipe. It is the mother of all pipes.'); - } - - public function getProperties() - { - return $this->properties; - } } diff --git a/src/Pipeline/Environment/EnvironmentInterface.php b/src/Pipeline/Environment/EnvironmentInterface.php index 1fe8a1f..6a10a75 100644 --- a/src/Pipeline/Environment/EnvironmentInterface.php +++ b/src/Pipeline/Environment/EnvironmentInterface.php @@ -1,94 +1,32 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ -namespace Driver\Pipeline\Environment; +declare(strict_types=1); -use Driver\Engines\MySql\Sandbox\Utilities; -use Driver\Pipeline\Stage\Factory as StageFactory; -use Driver\System\YamlFormatter; +namespace Driver\Pipeline\Environment; interface EnvironmentInterface { - public function __construct($name, array $properties, Utilities $utilities); - /** - * @return array + * @return string[] */ public function getOnlyForPipeline(): array; - /** - * @return array - */ - public function getFiles(); + public function getName(): string; /** - * @param string $type - * @param string $path - * @return void + * @return string[] */ - public function addFile($type, $path); + public function getTransformations(): array; - /** - * @return string - */ - public function getName(); - - /** - * @param $key - * @return string - */ - public function getData($key); - - /** - * @return array - */ - public function getAllData(); - - /** - * @return array - */ - public function getTransformations(); - - /** - * @return int - */ - public function getSort(); - - /** - * @return array - */ - public function getIgnoredTables(); - - /** - * @param $tableName - * @return void - */ - public function addIgnoredTable($tableName); + public function getSort(): int; /** - * @return array + * @return string[] */ - public function getEmptyTables(); + public function getIgnoredTables(): array; /** - * @param $tableName - * @return void + * @return string[] */ - public function addEmptyTable($tableName); + public function getEmptyTables(): array; } diff --git a/src/Pipeline/Environment/Factory.php b/src/Pipeline/Environment/Factory.php index 8345028..03cdf60 100644 --- a/src/Pipeline/Environment/Factory.php +++ b/src/Pipeline/Environment/Factory.php @@ -1,48 +1,32 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/29/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline\Environment; use DI\Container; use Driver\Engines\MySql\Sandbox\Utilities; -use Driver\Pipeline\Master; +use Driver\Pipeline\Exception\PipeLineNotFound; use Driver\System\Configuration; -use Driver\System\Factory\FactoryInterface; class Factory { - const DEFAULT_ENV = 'default'; + private const DEFAULT_ENV = 'default'; - private $configuration; - private $container; - private $type; - private $utilities; + private Configuration $configuration; + private Container $container; + private Utilities $utilities; + private string $type; - public function __construct(Configuration $configuration, Container $container, Utilities $utilities, $type) + public function __construct(Configuration $configuration, Container $container, Utilities $utilities, string $type) { $this->configuration = $configuration; $this->container = $container; - $this->type = $type; $this->utilities = $utilities; + $this->type = $type; } - public function createDefault() + public function createDefault(): EnvironmentInterface { return $this->container->make($this->type, [ 'name' => self::DEFAULT_ENV, @@ -50,11 +34,7 @@ public function createDefault() ]); } - /** - * @param $name - * @return EnvironmentInterface - */ - public function create($name) + public function create(string $name): EnvironmentInterface { return $this->container->make($this->type, [ 'name' => $name, @@ -63,17 +43,18 @@ public function create($name) ]); } - protected function environmentExists($name) - { - return is_array($this->configuration->getNode("environments/{$name}")); - } - - protected function getEnvironmentProperties($name) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + private function getEnvironmentProperties(string $name): array { if ($this->environmentExists($name)) { return $this->configuration->getNode("environments/{$name}"); } else { - throw new \Driver\Pipeline\Exception\PipeLineNotFound(); + throw new PipeLineNotFound(); } } -} \ No newline at end of file + + private function environmentExists(string $name): bool + { + return is_array($this->configuration->getNode("environments/{$name}")); + } +} diff --git a/src/Pipeline/Environment/Manager.php b/src/Pipeline/Environment/Manager.php index 87413b2..23b4d70 100644 --- a/src/Pipeline/Environment/Manager.php +++ b/src/Pipeline/Environment/Manager.php @@ -1,32 +1,21 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/10/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline\Environment; +use ArrayIterator; use Driver\System\Configuration; -use Haystack\HArray; + +use function array_filter; +use function array_keys; +use function array_map; class Manager { - const ALL_ENVIRONMENTS = 'all'; + private const ALL_ENVIRONMENTS = 'all'; - private \ArrayIterator $runFor; + private ArrayIterator $runFor; private Factory $factory; private Configuration $configuration; private bool $hasCustomRunList = false; @@ -35,14 +24,18 @@ public function __construct(Factory $factory, Configuration $configuration) { $this->factory = $factory; $this->configuration = $configuration; - $this->runFor = new \ArrayIterator([]); + $this->runFor = new ArrayIterator([]); } - public function setRunFor($environments) + /** + * @param string[]|string $environments + */ + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingNativeTypeHint + public function setRunFor($environments): void { - if (strtolower($environments) === self::ALL_ENVIRONMENTS || !$environments) { + if (!$environments || strtolower($environments) === self::ALL_ENVIRONMENTS) { $environmentList = $this->getAllEnvironments(); - } else if (is_string($environments)) { + } elseif (is_string($environments)) { $environmentList = array_filter(explode(',', $environments)); array_walk($environmentList, 'trim'); $this->hasCustomRunList = true; @@ -50,26 +43,30 @@ public function setRunFor($environments) $environmentList = $environments; } - $this->runFor = new \ArrayIterator($this->mapNamesToEnvironments($environmentList)); + $this->runFor = new ArrayIterator($this->mapNamesToEnvironments($environmentList)); } - public function getAllEnvironments() + /** + * @return string[] + */ + public function getAllEnvironments(): array { - $output = (new HArray($this->configuration->getNode('environments'))) - ->filter(function($value) { - return !isset($value['empty']) || $value['empty'] == false; - })->toArray(); - - return array_keys($output); + return array_keys(array_filter($this->configuration->getNode('environments'), function ($value) { + return !isset($value['empty']) || $value['empty'] == false; + })); } - private function mapNamesToEnvironments(array $environments) + /** + * @param string[] $environments + * @return EnvironmentInterface[] + */ + private function mapNamesToEnvironments(array $environments): array { - $mapped = (new HArray($environments))->map(function($name) { + $mapped = array_map(function ($name) { return $this->factory->create($name); - })->toArray(); + }, $environments); - usort($mapped, function(EnvironmentInterface $a, EnvironmentInterface $b) { + usort($mapped, function (EnvironmentInterface $a, EnvironmentInterface $b) { if ($a->getSort() == $b->getSort()) { return 0; } @@ -80,10 +77,7 @@ private function mapNamesToEnvironments(array $environments) return $mapped; } - /** - * @return \ArrayIterator - */ - public function getRunFor() + public function getRunFor(): ArrayIterator { return $this->runFor; } diff --git a/src/Pipeline/Environment/Primary.php b/src/Pipeline/Environment/Primary.php index 09ee778..07b27bf 100644 --- a/src/Pipeline/Environment/Primary.php +++ b/src/Pipeline/Environment/Primary.php @@ -1,100 +1,58 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/29/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline\Environment; use Driver\Engines\MySql\Sandbox\Utilities; -use Driver\Pipeline\Stage\Factory as StageFactory; -use Driver\Pipeline\Transport\Status; -use Driver\System\YamlFormatter; -use Haystack\HArray; -use Icicle\Concurrent\Worker\Environment; class Primary implements EnvironmentInterface { - private $name; - private $properties; - private $files; - private $ignoredTables; - private $emptyTables; - - /** @var Utilities $utilities */ - private $utilities; - - public function __construct($name, array $properties, Utilities $utilities) + private string $name; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $properties; + private Utilities $utilities; + /** @var string[] */ + private array $ignoredTables = []; + /** @var string[] */ + private array $emptyTables = []; + + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function __construct(string $name, array $properties, Utilities $utilities) { - $this->properties = $properties; $this->name = $name; + $this->properties = $properties; $this->utilities = $utilities; } - public function addFile($type, $path) - { - $this->files[$type] = $path; - } - + /** + * @return string[] + */ public function getOnlyForPipeline(): array { return $this->properties['only_for_pipeline'] ?? []; } - public function getFiles() - { - return $this->files; - } - - public function getName() + public function getName(): string { return $this->name; } - public function getData($key) - { - return $this->properties[$key]; - } - - public function getAllData() - { - return $this->properties; - } - - public function addIgnoredTable($tableName): void - { - $this->ignoredTables[] = $tableName; - } - - public function addEmptyTable($tableName): void - { - $this->emptyTables[] = $tableName; - } - + /** + * @return string[] + */ public function getIgnoredTables(): array { if (!$this->ignoredTables) { - $this->ignoredTables = isset($this->properties['ignored_tables']) - ? $this->properties['ignored_tables'] - : []; + $this->ignoredTables = $this->properties['ignored_tables'] ?? []; } return $this->ignoredTables; } + /** + * @return string[] + */ public function getEmptyTables(): array { if (!$this->emptyTables) { @@ -114,6 +72,9 @@ public function getSort(): int : 1000; } + /** + * @return string[] + */ public function getTransformations(): array { return isset($this->properties['transformations']) @@ -121,26 +82,30 @@ public function getTransformations(): array : []; } - public function getAnonymizations(): array - { - return isset($this->properties['anonymize']) - ? $this->properties['anonymize'] - : []; - } - - private function flattenTransformations($input) + /** + * @param array $input + * @return string[] + */ + private function flattenTransformations(array $input): array { $output = []; - array_walk($input, function($transformations, $tableName) use (&$output) { - $output = array_merge($output, $this->parseVariables($this->utilities->tableName($tableName), $transformations)); + array_walk($input, function ($transformations, $tableName) use (&$output): void { + $output = array_merge( + $output, + $this->parseVariables($this->utilities->tableName($tableName), $transformations) + ); }); return $output; } - protected function parseVariables($tableName, array $input) + /** + * @param string[] $input + * @return string[] + */ + private function parseVariables(string $tableName, array $input): array { - return array_map(function($query) use ($tableName) { + return array_map(function ($query) use ($tableName) { return str_replace("{{table_name}}", $tableName, $query); }, $input); } diff --git a/src/Pipeline/Exception/EnvironmentNotFound.php b/src/Pipeline/Exception/EnvironmentNotFound.php deleted file mode 100644 index 6436a8c..0000000 --- a/src/Pipeline/Exception/EnvironmentNotFound.php +++ /dev/null @@ -1,25 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/29/16 - * @package default - **/ - -namespace Driver\Pipeline\Exception; - -class EnvironmentLineNotFound extends \Exception -{ - -} \ No newline at end of file diff --git a/src/Pipeline/Exception/PipeLineNotFound.php b/src/Pipeline/Exception/PipeLineNotFound.php index f62fdba..cd5808b 100644 --- a/src/Pipeline/Exception/PipeLineNotFound.php +++ b/src/Pipeline/Exception/PipeLineNotFound.php @@ -1,25 +1,9 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/29/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline\Exception; class PipeLineNotFound extends \Exception { - -} \ No newline at end of file +} diff --git a/src/Pipeline/Master.php b/src/Pipeline/Master.php index 347fa23..7ad3615 100644 --- a/src/Pipeline/Master.php +++ b/src/Pipeline/Master.php @@ -11,7 +11,7 @@ class Master { - const DEFAULT_NODE = 'build'; + public const DEFAULT_NODE = 'build'; private Configuration $configuration; private PipeLineSpanFactory $pipeLineFactory; @@ -27,11 +27,12 @@ public function __construct( $this->transportFactory = $transportFactory; } - public function run(string $set): void + public function run(string $set): TransportInterface { $pipeline = $this->pipeLineFactory->create($set); $transport = $pipeline($this->createTransport($set)); $pipeline->cleanup($transport); + return $transport; } protected function createTransport(string $set): TransportInterface diff --git a/src/Pipeline/PipeInterface.php b/src/Pipeline/PipeInterface.php deleted file mode 100644 index 80efe25..0000000 --- a/src/Pipeline/PipeInterface.php +++ /dev/null @@ -1,28 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/8/16 - * @package default - **/ - -namespace Driver\Pipeline; - -use Driver\Pipeline\Transport\TransportInterface; - -interface PipeInterface -{ - public function __construct(TransportInterface $transport); - public function execute(); -} \ No newline at end of file diff --git a/src/Pipeline/Span/Factory.php b/src/Pipeline/Span/Factory.php index 7e1dd98..2b1ed7d 100644 --- a/src/Pipeline/Span/Factory.php +++ b/src/Pipeline/Span/Factory.php @@ -1,67 +1,44 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/29/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline\Span; use DI\Container; +use Driver\Pipeline\Exception\PipeLineNotFound; use Driver\Pipeline\Master; use Driver\System\Configuration; -use Driver\System\Factory\FactoryInterface; class Factory { - private $configuration; - private $container; - private $type; + private Configuration $configuration; + private Container $container; + private string $type; - public function __construct(Configuration $configuration, Container $container, $type) + public function __construct(Configuration $configuration, Container $container, string $type) { $this->configuration = $configuration; $this->container = $container; $this->type = $type; } - /** - * @param $pipelineName - * @return SpanInterface - */ - public function create($pipelineName) - { - return $this->container->make($this->type, [ 'list' => $this->getNamedPipeline($pipelineName) ]); - } - - protected function getDefaultPipeline() + public function create(string $pipelineName): SpanInterface { - return $this->getNamedPipeline(Master::DEFAULT_NODE); + return $this->container->make($this->type, ['list' => $this->getNamedPipeline($pipelineName)]); } - protected function pipelineExists($name) + protected function pipelineExists(string $name): bool { return is_array($this->configuration->getNode("pipelines/{$name}")); } - protected function getNamedPipeline($name) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + protected function getNamedPipeline(string $name): array { if ($this->pipelineExists($name)) { return $this->configuration->getNode("pipelines/{$name}"); } else { - throw new \Driver\Pipeline\Exception\PipeLineNotFound(); + throw new PipeLineNotFound(); } } -} \ No newline at end of file +} diff --git a/src/Pipeline/Span/Primary.php b/src/Pipeline/Span/Primary.php index e37982e..d70ea32 100644 --- a/src/Pipeline/Span/Primary.php +++ b/src/Pipeline/Span/Primary.php @@ -1,24 +1,10 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/29/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline\Span; +use ArrayIterator; use Driver\Pipeline\Environment\EnvironmentInterface; use Driver\Pipeline\Environment\Factory as EnvironmentFactory; use Driver\Pipeline\Environment\Manager as EnvironmentManager; @@ -27,29 +13,35 @@ use Driver\Pipeline\Transport\Status; use Driver\Pipeline\Transport\TransportInterface; use Driver\System\YamlFormatter; -use Haystack\HArray; -use Symfony\Component\Console\Output\OutputInterface; + +use function array_filter; +use function array_walk; class Primary implements SpanInterface { - const PIPE_SET_NODE = 'parent'; - const UNSET_ENVIRONMENT = 'default'; - - private $stages; - private $stageFactory; - private $environmentManager; - private $environmentFactory; - - public function __construct(array $list, StageFactory $stageFactory, YamlFormatter $yamlFormatter, EnvironmentManager $environmentManager, EnvironmentFactory $environmentFactory) - { + private const PIPE_SET_NODE = 'parent'; + + /** @var StageInterface[] */ + private array $stages; + private StageFactory $stageFactory; + private EnvironmentManager $environmentManager; + private EnvironmentFactory $environmentFactory; + + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function __construct( + array $list, + StageFactory $stageFactory, + YamlFormatter $yamlFormatter, + EnvironmentManager $environmentManager, + EnvironmentFactory $environmentFactory + ) { $this->stageFactory = $stageFactory; $this->environmentManager = $environmentManager; $this->environmentFactory = $environmentFactory; - $this->stages = $this->generateStageMap($yamlFormatter->extractSpanList($list)); } - public function __invoke(TransportInterface $transport, $testMode = false) + public function __invoke(TransportInterface $transport, bool $testMode = false): TransportInterface { $stages = !$testMode ? $this->stages : []; @@ -61,26 +53,29 @@ public function __invoke(TransportInterface $transport, $testMode = false) return $transport->withStatus(new Status(self::PIPE_SET_NODE, 'complete')); } - public function cleanup(TransportInterface $transport, $testMode = false) + public function cleanup(TransportInterface $transport, bool $testMode = false): TransportInterface { $stages = !$testMode ? $this->stages : []; - (new HArray($stages)) - ->walk(function(StageInterface $stage) use ($transport){ - return $stage->cleanup($transport); - }); + array_walk($stages, function (StageInterface $stage) use ($transport) { + return $stage->cleanup($transport); + }); return $transport->withStatus(new Status(self::PIPE_SET_NODE, 'cleaned')); } - private function generateStageMap($list) + /** + * @return StageInterface[] + */ + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + private function generateStageMap(array $list): array { $stages = $this->mapToStageObjects($this->sortStages($this->filterStages($list))); - $output = new \ArrayIterator(); - $defaultEnvironment = new \ArrayIterator([ $this->environmentFactory->createDefault() ]); + $output = new ArrayIterator(); + $defaultEnvironment = new ArrayIterator([ $this->environmentFactory->createDefault() ]); - array_walk($stages, function(StageInterface $stage) use (&$output, $defaultEnvironment) { + array_walk($stages, function (StageInterface $stage) use (&$output, $defaultEnvironment): void { if ($stage->isRepeatable()) { $output = $this->repeatForEnvironments($stage, $output, $this->environmentManager->getRunFor()); } else { @@ -97,9 +92,10 @@ private function generateStageMap($list) return $output->getArrayCopy(); } - private function sortStages(array $stages) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification,SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + private function sortStages(array $stages): array { - uasort($stages, function($a, $b) { + uasort($stages, function ($a, $b) { if ($a['sort'] == $b['sort']) { return 0; } @@ -109,36 +105,43 @@ private function sortStages(array $stages) return $stages; } - private function filterStages(array $stages) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification,SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + private function filterStages(array $stages): array { - return (new HArray($stages)) - ->filter(function($actions) { - return count($actions) > 0; - })->toArray(); + return array_filter($stages, function ($actions) { + return count($actions) > 0; + }); } - private function mapToStageObjects(array $stages) + /** + * @return StageInterface[] $stages + */ + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + private function mapToStageObjects(array $stages): array { - return array_filter(array_map(function($stage, $name) { - if (isset($stage['actions'])) { - return $this->stageFactory->create($stage['actions'], $name); - } else { - return null; - } + return array_filter(array_map(function ($stage, $name) { + return isset($stage['actions']) ? $this->stageFactory->create($stage['actions'], $name) : null; }, array_values($stages), array_keys($stages))); } - private function repeatForEnvironments(StageInterface $stage, \ArrayIterator $output, \ArrayIterator $environments) - { - return array_reduce($environments->getArrayCopy(), function(\ArrayIterator $input, EnvironmentInterface $environment) use ($stage) { - $output = new \ArrayIterator($input->getArrayCopy()); - $output->append($stage->withEnvironment($environment)); - - return $output; - }, $output); + private function repeatForEnvironments( + StageInterface $stage, + ArrayIterator $output, + ArrayIterator $environments + ): ArrayIterator { + return array_reduce( + $environments->getArrayCopy(), + function (ArrayIterator $input, EnvironmentInterface $environment) use ($stage) { + $output = new ArrayIterator($input->getArrayCopy()); + $output->append($stage->withEnvironment($environment)); + + return $output; + }, + $output + ); } - private function verifyTransport(TransportInterface $transport, $lastCommand) + private function verifyTransport(?TransportInterface $transport, string $lastCommand): TransportInterface { if (!$transport) { throw new \Exception('No Transport object was returned from the last command executed: ' . $lastCommand); diff --git a/src/Pipeline/Span/SpanInterface.php b/src/Pipeline/Span/SpanInterface.php index 3e63f21..a425761 100644 --- a/src/Pipeline/Span/SpanInterface.php +++ b/src/Pipeline/Span/SpanInterface.php @@ -1,34 +1,14 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline\Span; -use Driver\Pipeline\Environment\Factory as EnvironmentFactory; -use Driver\Pipeline\Environment\Manager; -use Driver\Pipeline\Stage\Factory as StageFactory; -use Driver\System\YamlFormatter; +use Driver\Pipeline\Transport\TransportInterface; interface SpanInterface { - public function __construct(array $list, StageFactory $stageFactory, YamlFormatter $yamlFormatter, Manager $environmentManager, EnvironmentFactory $environmentFactory); - - public function __invoke(\Driver\Pipeline\Transport\TransportInterface $transport); + public function __invoke(TransportInterface $transport): TransportInterface; - public function cleanup(\Driver\Pipeline\Transport\TransportInterface $transport); -} \ No newline at end of file + public function cleanup(TransportInterface $transport): TransportInterface; +} diff --git a/src/Pipeline/Stage/Factory.php b/src/Pipeline/Stage/Factory.php index 534e873..b3af467 100644 --- a/src/Pipeline/Stage/Factory.php +++ b/src/Pipeline/Stage/Factory.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/29/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline\Stage; @@ -24,23 +9,20 @@ class Factory { - private $configuration; - private $container; - private $type; + private Configuration $configuration; + private Container $container; + private string $type; - public function __construct(Configuration $configuration, Container $container, $type) + public function __construct(Configuration $configuration, Container $container, string $type) { $this->configuration = $configuration; $this->container = $container; $this->type = $type; } - /** - * @param $actions - * @return StageInterface - */ - public function create($actions, $name) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function create(array $actions, string $name): StageInterface { - return $this->container->make($this->type, [ 'actions' => $actions, 'name' => $name ]); + return $this->container->make($this->type, ['actions' => $actions, 'name' => $name]); } -} \ No newline at end of file +} diff --git a/src/Pipeline/Stage/Primary.php b/src/Pipeline/Stage/Primary.php index c7c01c8..6902e4a 100644 --- a/src/Pipeline/Stage/Primary.php +++ b/src/Pipeline/Stage/Primary.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/29/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline\Stage; @@ -23,132 +8,140 @@ use Driver\Commands\CommandInterface; use Driver\Commands\ErrorInterface; use Driver\Commands\Factory as CommandFactory; -use Driver\Pipeline\Command; use Driver\Pipeline\Environment\EnvironmentInterface; use Driver\Pipeline\Transport\Status; use Driver\Pipeline\Transport\TransportInterface; -use Driver\System\YamlFormatter; -use GuzzleHttp\Promise\Promise; -use Haystack\HArray; + +use function array_reduce; class Primary implements StageInterface { - const PIPE_SET_NODE = 'parent'; - const REPEAT_PREFIX = 'repeat'; - const EMPTY_NODE = 'empty'; - - private $actions; - private $commandFactory; - private $environment; - private $name; - - public function __construct(array $actions, $name, CommandFactory $commandFactory, EnvironmentInterface $environment = null) - { + private const PIPE_SET_NODE = 'parent'; + private const REPEAT_PREFIX = 'repeat'; + private const EMPTY_NODE = 'empty'; + + /** @var CommandInterface[] */ + private array $actions; + private string $name; + private CommandFactory $commandFactory; + private ?EnvironmentInterface $environment; + + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function __construct( + array $actions, + string $name, + CommandFactory $commandFactory, + EnvironmentInterface $environment = null + ) { $this->commandFactory = $commandFactory; $this->actions = $this->sortActions($this->initActions($actions)); $this->name = $name; $this->environment = $environment; } - private function initActions($actions) + public function __invoke(TransportInterface $transport, bool $testMode = false): TransportInterface { - return array_map(function($properties) { - if (is_a($properties, CommandInterface::class)) { - return $properties; - } else { - if ($properties['name'] !== self::EMPTY_NODE) { - return $this->commandFactory->create($properties['name'], $properties); - } + $actions = !$testMode ? $this->actions : []; + + $transport = array_reduce($actions, function (TransportInterface $transport, CommandInterface $command) { + return $this->runCommand($transport, $command); + }, $transport); + + return $transport->withStatus(new Status(self::PIPE_SET_NODE, 'complete')); + } + + public function cleanup(TransportInterface $transport, bool $testMode = false): TransportInterface + { + $actions = !$testMode ? $this->actions : []; + + $transport = array_reduce($actions, function (TransportInterface $transport, CommandInterface $command) { + /** @var CleanupInterface $command */ + if (is_a($command, CleanupInterface::class)) { + $command->cleanup($transport, $this->environment); } - }, $actions); + return $transport; + }, $transport); + + return $transport->withStatus(new Status(self::PIPE_SET_NODE, 'cleaned')); } - public function isRepeatable() + public function isRepeatable(): bool { return substr($this->name, 0, strlen(self::REPEAT_PREFIX)) === self::REPEAT_PREFIX; } - public function withEnvironment(EnvironmentInterface $environment) + public function withEnvironment(EnvironmentInterface $environment): StageInterface { return new self($this->actions, $this->name, $this->commandFactory, $environment); } - public function getName() + public function getName(): string { return $this->name; } - public function getEnvironment() + public function getEnvironment(): ?EnvironmentInterface { return $this->environment; } - public function __invoke(TransportInterface $transport, $testMode = false) - { - $actions = !$testMode ? $this->actions : []; - - $transport = array_reduce($actions, function(TransportInterface $transport, CommandInterface $command) { - return $this->runCommand($transport, $command); - }, $transport); - - return $transport->withStatus(new Status(self::PIPE_SET_NODE, 'complete')); - } - - public function cleanup(TransportInterface $transport, $testMode = false) + /** + * @return CommandInterface[] + */ + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + private function initActions(array $actions): array { - $actions = !$testMode ? $this->actions : []; - - $transport = array_reduce($actions, function(TransportInterface $transport, CommandInterface $command) { - /** @var CleanupInterface $command */ - if (is_a($command, CleanupInterface::class)) { - $command->cleanup($transport, $this->environment); + return array_map(function ($properties) { + if (is_a($properties, CommandInterface::class)) { + return $properties; + } else { + if ($properties['name'] !== self::EMPTY_NODE) { + return $this->commandFactory->create($properties['name'], $properties); + } } - return $transport; - }, $transport); - - return $transport->withStatus(new Status(self::PIPE_SET_NODE, 'cleaned')); + }, $actions); } - private function runCommand(TransportInterface $transport, CommandInterface $command) + private function runCommand(TransportInterface $transport, CommandInterface $command): TransportInterface { try { if (!$this->hasError($transport)) { return $this->verifyTransport($command->go($transport, $this->environment), $command); - } else if ($this->hasErrorHandler($command)) { + } elseif ($this->hasErrorHandler($command)) { return $this->verifyTransport($command->error($transport, $this->environment), $command); } else { return $transport; } } catch (\Exception $ex) { - return $transport->withStatus(new Status($command, $ex->getMessage(), true)); + return $transport->withStatus(new Status(get_class($command), $ex->getMessage(), true)); } } - private function hasErrorHandler(CommandInterface $command) + private function hasErrorHandler(CommandInterface $command): bool { return is_a($command, ErrorInterface::class); } - private function hasError(TransportInterface $transport) + private function hasError(TransportInterface $transport): bool { - return (new HArray($transport->getStatuses()))->reduce(function($carry, Status $status) { + return array_reduce($transport->getStatuses(), function (bool $carry, Status $status): bool { return $carry || $status->isError(); }, false); } - private function sortActions($actions) + /** + * @param CommandInterface[] $actions + * @return CommandInterface[] + */ + private function sortActions(array $actions): array { $actions = array_filter($actions); - $getSort = function(CommandInterface $command) { + $getSort = function (CommandInterface $command) { $properties = $command->getProperties(); - if (isset($properties['sort'])) { - return $properties['sort']; - } else { - return 1000; - } + return $properties['sort'] ?? 1000; }; - usort($actions, function(CommandInterface $a, CommandInterface $b) use ($getSort) { + usort($actions, function (CommandInterface $a, CommandInterface $b) use ($getSort) { if ($getSort($a) == $getSort($b)) { return 0; } @@ -158,10 +151,12 @@ private function sortActions($actions) return $actions; } - private function verifyTransport(TransportInterface $transport, CommandInterface $command) + private function verifyTransport(TransportInterface $transport, CommandInterface $command): TransportInterface { if (!$transport) { - throw new \Exception('No Transport object was returned from the last command executed: ' . get_class($command)); + throw new \Exception( + 'No Transport object was returned from the last command executed: ' . get_class($command) + ); } return $transport; diff --git a/src/Pipeline/Stage/StageInterface.php b/src/Pipeline/Stage/StageInterface.php index 0ec5432..d842b9e 100644 --- a/src/Pipeline/Stage/StageInterface.php +++ b/src/Pipeline/Stage/StageInterface.php @@ -1,38 +1,21 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline\Stage; -use Driver\Commands\Factory as CommandFactory; use Driver\Pipeline\Environment\EnvironmentInterface; +use Driver\Pipeline\Transport\TransportInterface; interface StageInterface { - public function __construct(array $list, $name, CommandFactory $commandFactory, EnvironmentInterface $environmentInterface); - - public function __invoke(\Driver\Pipeline\Transport\TransportInterface $transport); + public function __invoke(TransportInterface $transport): TransportInterface; - public function cleanup(\Driver\Pipeline\Transport\TransportInterface $transport); + public function cleanup(TransportInterface $transport): TransportInterface; - public function getName(); + public function getName(): string; - public function withEnvironment(EnvironmentInterface $environment); + public function withEnvironment(EnvironmentInterface $environment): StageInterface; - public function isRepeatable(); + public function isRepeatable(): bool; } diff --git a/src/Pipeline/Transport/Error.php b/src/Pipeline/Transport/Error.php deleted file mode 100644 index de4ce0b..0000000 --- a/src/Pipeline/Transport/Error.php +++ /dev/null @@ -1,35 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/29/16 - * @package default - **/ - -namespace Driver\Pipeline\Transport; - -class Error -{ - private $message; - - public function __construct($message) - { - $this->message = $message; - } - - public function getMessage() - { - return $this->message; - } -} \ No newline at end of file diff --git a/src/Pipeline/Transport/Factory.php b/src/Pipeline/Transport/Factory.php index cd78c9a..a1e6381 100644 --- a/src/Pipeline/Transport/Factory.php +++ b/src/Pipeline/Transport/Factory.php @@ -1,46 +1,24 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/29/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline\Transport; -use Driver\Pipeline\Transport; -use Driver\System\Configuration; use Driver\System\Logs\LoggerInterface; -use Driver\Pipeline\Environment\Factory as EnvironmentFactory; class Factory { - private $configuration; - private $type; - private $logger; - private $environmentFactory; + private string $type; + private LoggerInterface $logger; - public function __construct(Configuration $configuration, $type, LoggerInterface $logger, EnvironmentFactory $environmentFactory) + public function __construct(string $type, LoggerInterface $logger) { - $this->configuration = $configuration; $this->type = $type; $this->logger = $logger; - $this->environmentFactory = $environmentFactory; } - public function create($pipeline) + public function create(string $pipeline): TransportInterface { - return new $this->type($pipeline, [], [], $this->environmentFactory->createDefault(), $this->logger); + return new $this->type($pipeline, $this->logger); } -} \ No newline at end of file +} diff --git a/src/Pipeline/Transport/Primary.php b/src/Pipeline/Transport/Primary.php index eaa76ce..5588ed7 100644 --- a/src/Pipeline/Transport/Primary.php +++ b/src/Pipeline/Transport/Primary.php @@ -1,59 +1,55 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/8/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline\Transport; -use Driver\Pipeline\Environment\EnvironmentInterface; use Driver\System\Logs\LoggerInterface; +use Driver\System\Logs\Primary as PrimaryLogger; + +use function array_merge; class Primary implements TransportInterface { - private $data; - private $statuses; - private $pipeline; - private $logger; - private $environment; + private string $pipeline; + private LoggerInterface $logger; + /** @var Status[] */ + private array $statuses; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification + private array $data; + /** + * @param Status[] $statuses + */ + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( - $pipeline, - $statuses = [], - $data = [], - EnvironmentInterface $environment = null, - LoggerInterface $logger = null + string $pipeline, + ?LoggerInterface $logger = null, + array $statuses = [], + array $data = [] ) { $this->pipeline = $pipeline; + $this->logger = $logger ?? new PrimaryLogger(); $this->statuses = $statuses; $this->data = $data; - $this->logger = $logger; - $this->environment = $environment; } - public function getErrors() + /** + * @return Status[] + */ + public function getErrors(): array { - return array_filter($this->statuses, function(Status $status) { + return array_filter($this->statuses, function (Status $status) { return $status->isError(); }); } - public function getErrorsByNode($node) + /** + * @return Status[] + */ + public function getErrorsByNode(string $node): array { - return array_filter($this->statuses, function(Status $status) use ($node) { + return array_filter($this->statuses, function (Status $status) use ($node) { return $status->isError() && $status->getNode() === $node; }); } @@ -63,39 +59,58 @@ public function getPipeline(): string return $this->pipeline; } - public function withStatus(Status $status) + public function withStatus(Status $status): self { - return new self($this->pipeline, array_merge($this->statuses, [ $status ]), $this->data, $this->environment, $this->logger); + return new self( + $this->pipeline, + $this->logger, + array_merge($this->statuses, [$status]), + $this->data + ); } - public function getStatuses() + /** + * @return Status[] + */ + public function getStatuses(): array { return $this->statuses; } - public function getStatusesByNode($node) + /** + * @return Status[] + */ + public function getStatusesByNode(string $node): array { - return array_filter($this->statuses, function(Status $status) use ($node) { + return array_filter($this->statuses, function (Status $status) use ($node) { return $status->getNode() === $node; }); } - public function withNewData($key, $value) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint + public function withNewData(string $key, $value): self { - return new self($this->pipeline, $this->statuses, array_merge($this->data, [$key => $value]), $this->environment, $this->logger); + return new self( + $this->pipeline, + $this->logger, + $this->statuses, + array_merge($this->data, [$key => $value]) + ); } - public function getAllData() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint + public function getAllData(): array { return $this->data; } - public function getData($key) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint + public function getData(string $key) { - return isset($this->data[$key]) ? $this->data[$key] : false; + return $this->data[$key] ?? false; } - public function getLogger() + public function getLogger(): LoggerInterface { return $this->logger; } diff --git a/src/Pipeline/Transport/Status.php b/src/Pipeline/Transport/Status.php index a8c22b2..70a142e 100644 --- a/src/Pipeline/Transport/Status.php +++ b/src/Pipeline/Transport/Status.php @@ -1,49 +1,34 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline\Transport; class Status { - private $node; - private $message; - private $isError; + private string $node; + private string $message; + private bool $isError; - public function __construct($node, $message, $isError = false) + public function __construct(string $node, string $message, bool $isError = false) { $this->node = $node; $this->message = $message; $this->isError = $isError; } - public function isError() + public function isError(): bool { - return (bool)$this->isError; + return $this->isError; } - public function getNode() + public function getNode(): string { return $this->node; } - public function getMessage() + public function getMessage(): string { return $this->message; } -} \ No newline at end of file +} diff --git a/src/Pipeline/Transport/TransportInterface.php b/src/Pipeline/Transport/TransportInterface.php index 7f3705a..017b862 100644 --- a/src/Pipeline/Transport/TransportInterface.php +++ b/src/Pipeline/Transport/TransportInterface.php @@ -1,94 +1,46 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Pipeline\Transport; -use Driver\Pipeline\Environment\EnvironmentInterface; use Driver\System\Logs\LoggerInterface; interface TransportInterface { - const STATUS_FAILED = 1; - const STATUS_SUCCESS = 2; - const STATUS_PENDING = 3; - - public function __construct( - $pipeline, - $statuses = [], - $data = [], - EnvironmentInterface $environment, - LoggerInterface $log = null - ); - - /** - * @return string - */ public function getPipeline(): string; /** - * @return array + * @return Status[] */ - public function getErrors(); + public function getErrors(): array; /** - * @param $node - * @return array + * @return Status[] */ - public function getErrorsByNode($node); + public function getErrorsByNode(string $node): array; /** - * @return array + * @return Status[] */ - public function getStatuses(); + public function getStatuses(): array; /** - * @param $node - * @return array + * @return Status[] */ - public function getStatusesByNode($node); + public function getStatusesByNode(string $node): array; - /** - * @return array - */ - public function getAllData(); + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint + public function getAllData(): array; - /** - * @param Status $status - * @return self - */ - public function withStatus(Status $status); + public function withStatus(Status $status): self; - /** - * @param string $key - * @return mixed - */ - public function getData($key); + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint + public function getData(string $key); - /** - * @param string $key - * @param string $value - * @return self - */ - public function withNewData($key, $value); + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint + public function withNewData(string $key, $value): self; - /** - * @return LoggerInterface - */ - public function getLogger(); + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint + public function getLogger(): LoggerInterface; } diff --git a/src/System/Application.php b/src/System/Application.php index 26d2ca8..61cfa7a 100644 --- a/src/System/Application.php +++ b/src/System/Application.php @@ -1,71 +1,31 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/8/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\System; use DI\Container; +use Driver\Pipeline\Command; use Symfony\Component\Console\Application as ConsoleApplication; -use Symfony\Component\Console\Event\ConsoleSignalEvent; -use Symfony\Component\EventDispatcher\EventDispatcher; -/** - * Class Application - * Main class to run functionality - * - * @package Driver\System - */ class Application { - protected $console; - protected $configuration; - protected $container; - - private EventDispatcher $eventDispatcher; - - const RUN_MODE_NORMAL = 'normal'; - const RUN_MODE_TEST = 'test'; + private ConsoleApplication $console; + private Container $container; - public function __construct( - ConsoleApplication $console, - Configuration $configuration, - Container $container, - EventDispatcher $eventDispatcher - ) { + public function __construct(ConsoleApplication $console, Container $container) + { $this->console = $console; - $this->configuration = $configuration; $this->container = $container; - $this->eventDispatcher = $eventDispatcher; } - public function run($mode = self::RUN_MODE_NORMAL) + public function run(): void { - foreach ($this->configuration->getNode('commands') as $name => $settings) { - $this->console->add($this->container->get($settings['class'])); - } - + $this->console->add($this->container->get(Command::class)); $this->console->run(); } - /** - * @return ConsoleApplication - */ - public function getConsole() + public function getConsole(): ConsoleApplication { return $this->console; } diff --git a/src/System/Arguments.php b/src/System/Arguments.php deleted file mode 100644 index c5b7078..0000000 --- a/src/System/Arguments.php +++ /dev/null @@ -1,39 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/10/16 - * @package default - **/ - -namespace Driver\System; - -class Arguments -{ - protected $data = array(); - - public function setData($key, $value) - { - $this->data[$key] = $value; - } - - public function getData($key) - { - if (isset($this->data[$key])) { - return $this->data[$key]; - } else { - return false; - } - } -} \ No newline at end of file diff --git a/src/System/AwsClientFactory.php b/src/System/AwsClientFactory.php index 20b1b1a..24cc496 100644 --- a/src/System/AwsClientFactory.php +++ b/src/System/AwsClientFactory.php @@ -1,28 +1,14 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/7/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\System; -use Aws\Rds\RdsClient; +use Aws\AwsClient; class AwsClientFactory { + /** @var callable|null */ private $creator; public function __construct(callable $creator = null) @@ -30,7 +16,8 @@ public function __construct(callable $creator = null) $this->creator = $creator; } - public function create($serviceType, $arguments) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + public function create(string $serviceType, array $arguments): AwsClient { if (!$this->creator || !is_callable($this->creator)) { return $this->doCreate($serviceType, $arguments); @@ -39,7 +26,8 @@ public function create($serviceType, $arguments) } } - private function doCreate($serviceType, $arguments) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification + private function doCreate(string $serviceType, array $arguments): AwsClient { if (strpos($serviceType, "\\") !== false) { $type = $serviceType; @@ -49,4 +37,4 @@ private function doCreate($serviceType, $arguments) return new $type($arguments); } -} \ No newline at end of file +} diff --git a/src/System/Configuration.php b/src/System/Configuration.php index 39bb65b..418fe0a 100755 --- a/src/System/Configuration.php +++ b/src/System/Configuration.php @@ -23,7 +23,9 @@ class Configuration { private FileCollector $fileCollector; private FileLoader $loader; + // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint private array $nodes = ['pipelines' => []]; + /** @var array */ private array $files = []; public function __construct(FileCollector $fileCollector, FileLoader $fileLoader) @@ -32,6 +34,7 @@ public function __construct(FileCollector $fileCollector, FileLoader $fileLoader $this->loader = $fileLoader; } + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint public function getNodes(): array { if (!count($this->files)) { @@ -43,27 +46,25 @@ public function getNodes(): array return $this->nodes; } - /** - * @return mixed - */ - public function getNode($node) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint + public function getNode(string $node) { $path = explode('/', $node); $nodes = $this->getNodes(); - return array_reduce($path, function($nodes, $item) { + return array_reduce($path, function ($nodes, $item) { return $nodes[$item] ?? null; }, $nodes); } - public function getNodeString($node): string + public function getNodeString(string $node): string { $value = $this->getNode($node); return is_string($value) ? $value : ''; } - private function loadConfigurationFor($file): void + private function loadConfigurationFor(string $file): void { if (!isset($this->files[$file])) { try { @@ -86,6 +87,7 @@ private function loadConfigurationFor($file): void } } + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint,SlevomatCodingStandard.TypeHints.ReturnTypeHint private function recursiveMerge(array $array1, array $array2): array { $merged = $array1; @@ -93,7 +95,7 @@ private function recursiveMerge(array $array1, array $array2): array foreach ($array2 as $key => $value) { if (is_array($value) && isset($merged[$key]) && is_array($merged[$key]) && !is_int($key)) { $merged[$key] = $this->recursiveMerge($merged[$key], $value); - } else if (is_int($key)) { + } elseif (is_int($key)) { $merged[] = $value; } else { $merged[$key] = $value; @@ -105,6 +107,7 @@ private function recursiveMerge(array $array1, array $array2): array /** * Special handling for pipelines as they don't exactly follow the key/value pattern. */ + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint,SlevomatCodingStandard.TypeHints.ReturnTypeHint private function mergePipelines(array $new): array { if (!isset($this->nodes['pipelines']) || !count($this->nodes['pipelines'])) { @@ -121,7 +124,7 @@ private function mergePipelines(array $new): array array_keys($new) ); - return array_reduce($pipelinesKeys, function($carry, $key) use ($new) { + return array_reduce($pipelinesKeys, function ($carry, $key) use ($new) { $existing = $this->nodes['pipelines']; if (!isset($new[$key])) { $carry[$key] = $existing; @@ -137,22 +140,24 @@ private function mergePipelines(array $new): array }, []); } - private function mergePipeline($existing, $new) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint,SlevomatCodingStandard.TypeHints.ReturnTypeHint + private function mergePipeline(array $existing, array $new): array { - array_walk($new, function($value) use (&$existing) { + array_walk($new, function ($value) use (&$existing): void { $existing = $this->mergeStageIntoPipeline($existing, $value); }); return $existing; } + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint,SlevomatCodingStandard.TypeHints.ReturnTypeHint private function mergeStageIntoPipeline(array $existing, array $newStage): array { $matched = false; $output = array_reduce( array_keys($existing), - function($carry, $existingKey) use ($existing, $newStage, &$matched) { + function ($carry, $existingKey) use ($existing, $newStage, &$matched) { $existingStage = $existing[$existingKey]; $currentMatch = isset($newStage['name']) && isset($existingStage['name']) diff --git a/src/System/Configuration/FolderCollectionFactory.php b/src/System/Configuration/FolderCollectionFactory.php index 1748397..657f86e 100644 --- a/src/System/Configuration/FolderCollectionFactory.php +++ b/src/System/Configuration/FolderCollectionFactory.php @@ -18,7 +18,7 @@ class FolderCollectionFactory { - const VENDOR_DIRECTORY = 'vendor'; + private const VENDOR_DIRECTORY = 'vendor'; /** * @param string[] $allowedFolders diff --git a/src/System/ConnectionLoaderInterface.php b/src/System/ConnectionLoaderInterface.php deleted file mode 100644 index de30ed2..0000000 --- a/src/System/ConnectionLoaderInterface.php +++ /dev/null @@ -1,13 +0,0 @@ -output = $output; } - public function test($onFailure): void + public function test(callable $onFailure): void { - return; } public function authorizeIp(): void { } - public function isAvailable(): bool { return true; @@ -44,7 +37,8 @@ public function isAvailable(): bool public function getDSN(): string { - return "mysql:host={$this->getHost()};dbname={$this->getDatabase()};port={$this->getPort()};charset={$this->getCharset()}"; + return "mysql:host={$this->getHost()};dbname={$this->getDatabase()};" + . "port={$this->getPort()};charset={$this->getCharset()}"; } public function getCharset(): string @@ -94,7 +88,8 @@ public function getPassword(): string return $this->getValue('password', true); } - private function getValue($key, $required) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint + private function getValue(string $key, bool $required) { $value = $this->configuration->getNode("connections/mysql_debug/{$key}"); if (is_array($value) && $required) { @@ -105,6 +100,7 @@ private function getValue($key, $required) return $value; } + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification public function getPreserve(): array { return $this->getValue('preserve', false) ?: []; diff --git a/src/System/DebugMode.php b/src/System/DebugMode.php index a126eec..a1734a2 100644 --- a/src/System/DebugMode.php +++ b/src/System/DebugMode.php @@ -1,15 +1,12 @@ getHost()};dbname={$this->getDatabase()};port={$this->getPort()};charset={$this->getCharset()}"; + return "mysql:host={$this->getHost()};dbname={$this->getDatabase()};" + . "port={$this->getPort()};charset={$this->getCharset()}"; } public function getCharset(): string @@ -79,7 +74,8 @@ public function getPassword(): string return $this->getValue('password', true); } - private function getValue($key, $required) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint + private function getValue(string $key, bool $required) { $value = $this->configuration->getNode("connections/mysql/{$key}"); if (is_array($value) && $required) { @@ -90,6 +86,7 @@ private function getValue($key, $required) return $value; } + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification public function getPreserve(): array { return $this->getValue('preserve', false) ?: []; diff --git a/src/System/DependencyConfig.php b/src/System/DependencyConfig.php index adcc32a..87d2859 100644 --- a/src/System/DependencyConfig.php +++ b/src/System/DependencyConfig.php @@ -1,28 +1,10 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/8/16 - * @package default - **/ declare(strict_types=1); namespace Driver\System; use DI; -use DI\Definition\Helper\DefinitionHelper; use Driver\Engines\LocalConnectionInterface; use Driver\Engines\MySql\Sandbox\Connection; use Driver\Engines\RemoteConnectionInterface; @@ -43,13 +25,11 @@ public function __construct(bool $isDebug = false) $this->isDebug = $isDebug; } - /** - * @return - */ + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification public function get(): array { return [ - LoggerInterface::class => DI\Factory(function() { + LoggerInterface::class => DI\Factory(function () { return new Primary(); }), Environment\EnvironmentInterface::class => DI\factory([Environment\Primary::class, 'create']), @@ -68,15 +48,4 @@ public function get(): array ) ]; } - - /** - * @return - */ - public function getForTests(): array - { - return array_merge( - $this->get(), - [ ] - ); - } } diff --git a/src/System/Entry.php b/src/System/Entry.php index 526da91..e8e8c5e 100755 --- a/src/System/Entry.php +++ b/src/System/Entry.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/8/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\System; @@ -23,9 +8,13 @@ class Entry { - static private $arguments; + /** @var string[]|null */ + private static ?array $arguments = null; - public static function go($arguments) + /** + * @param string[] $arguments + */ + public static function go(array $arguments): void { set_time_limit(0); @@ -35,7 +24,7 @@ public static function go($arguments) self::$arguments = $arguments; self::configureDebug(); - $containerBuilder = new ContainerBuilder; + $containerBuilder = new ContainerBuilder(); $containerBuilder->addDefinitions((new DependencyConfig(self::isDebug()))->get()); $container = $containerBuilder->build(); @@ -43,19 +32,18 @@ public static function go($arguments) $application->run(); } - private static function configureDebug() + private static function configureDebug(): void { if (self::isDebug()) { error_reporting(E_ALL); - ini_set('display_errors', 1); + ini_set('display_errors', '1'); } } - private static function isDebug() + private static function isDebug(): bool { - return count(array_filter(self::$arguments, function($argument) { + return count(array_filter(self::$arguments, function ($argument) { return strpos($argument, '--debug') !== false; })) > 0; } } - diff --git a/src/System/Factory/Base.php b/src/System/Factory/Base.php deleted file mode 100644 index f98ba7e..0000000 --- a/src/System/Factory/Base.php +++ /dev/null @@ -1,29 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/29/16 - * @package default - **/ - -namespace Driver\System\Factory; - -class Base -{ - public function create($class, ...$args) - { - $class = new \ReflectionClass($class); - return $class->newInstanceArgs($args); - } -} \ No newline at end of file diff --git a/src/System/Factory/FactoryInterface.php b/src/System/Factory/FactoryInterface.php deleted file mode 100644 index 33a8825..0000000 --- a/src/System/Factory/FactoryInterface.php +++ /dev/null @@ -1,25 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ - -namespace Driver\System\Factory; - -interface FactoryInterface -{ - public function create(PropertyInterface $settings); -} \ No newline at end of file diff --git a/src/System/Factory/PropertyInterface.php b/src/System/Factory/PropertyInterface.php deleted file mode 100644 index 69cf61c..0000000 --- a/src/System/Factory/PropertyInterface.php +++ /dev/null @@ -1,25 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ - -namespace Driver\System\Factory; - -interface PropertyInterface -{ - -} \ No newline at end of file diff --git a/src/System/LocalConnectionLoader.php b/src/System/LocalConnectionLoader.php index 559879e..bad3b4f 100644 --- a/src/System/LocalConnectionLoader.php +++ b/src/System/LocalConnectionLoader.php @@ -1,9 +1,6 @@ get()->getConnection(); } - private function get() + private function get(): ConnectionInterface { if ($this->connection) { return $this->connection; @@ -72,7 +62,7 @@ private function get() return $this->connection; } - private function getDefault() + private function getDefault(): DefaultConnection { return $this->defaultConnection; } @@ -117,6 +107,7 @@ public function getPassword(): string return $this->get()->getPassword(); } + // phpcs:ignore SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification public function getPreserve(): array { return $this->get()->getPreserve(); diff --git a/src/System/Logs/LoggerInterface.php b/src/System/Logs/LoggerInterface.php index c790f84..40bac91 100644 --- a/src/System/Logs/LoggerInterface.php +++ b/src/System/Logs/LoggerInterface.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/25/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\System\Logs; @@ -25,5 +10,5 @@ interface LoggerInterface extends BaseLoggerInterface { - public function setParams(InputInterface $input, OutputInterface $output); -} \ No newline at end of file + public function setParams(InputInterface $input, OutputInterface $output): void; +} diff --git a/src/System/Logs/Primary.php b/src/System/Logs/Primary.php index 4267410..53ba1ed 100644 --- a/src/System/Logs/Primary.php +++ b/src/System/Logs/Primary.php @@ -1,25 +1,9 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/25/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\System\Logs; -use Driver\System\Logs\LoggerInterface; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Logger\ConsoleLogger; @@ -27,10 +11,8 @@ class Primary implements LoggerInterface { - /** - * @var array - */ - private $logLevelMap = [ + /** @var array */ + private array $logLevelMap = [ 1 => LogLevel::DEBUG, 2 => LogLevel::INFO, 5 => LogLevel::NOTICE, @@ -39,24 +21,11 @@ class Primary implements LoggerInterface 10 => LogLevel::CRITICAL ]; - /** - * @var InputInterface $input - */ - private $input; - /** - * @var OutputInterface $output - */ - private $output; - - /** - * @var ConsoleLogger $consoleLogger - */ - private $consoleLogger; - - /** - * @return ConsoleLogger - */ - private function getConsoleLogger() + private ?ConsoleLogger $consoleLogger = null; + private ?InputInterface $input = null; + private ?OutputInterface $output = null; + + private function getConsoleLogger(): ConsoleLogger { if (!$this->consoleLogger && $this->input && $this->output) { $this->consoleLogger = new ConsoleLogger($this->output, [], []); @@ -65,71 +34,72 @@ private function getConsoleLogger() return $this->consoleLogger; } - public function setParams(InputInterface $input, OutputInterface $output) + public function setParams(InputInterface $input, OutputInterface $output): void { $this->input = $input; $this->output = $output; } - public function emergency($message, array $context = array()) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint + public function emergency($message, array $context = array()): void { $this->log(10, $message); } - public function alert($message, array $context = array()) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint + public function alert($message, array $context = array()): void { $this->log(10, $message); } - public function critical($message, array $context = array()) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint + public function critical($message, array $context = array()): void { $this->log(10, $message); } - public function error($message, array $context = array()) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint + public function error($message, array $context = array()): void { $this->log(9, $message); } - public function warning($message, array $context = array()) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint + public function warning($message, array $context = array()): void { $this->log(8, $message); } - public function notice($message, array $context = array()) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint + public function notice($message, array $context = array()): void { $this->log(5, $message); } - public function info($message, array $context = array()) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint + public function info($message, array $context = array()): void { $this->log(2, $message); } - public function debug($message, array $context = array()) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint + public function debug($message, array $context = array()): void { $this->log(1, $message); } - /** - * @param int $level - * @param string $message - * @param array $context - */ - public function log($level, $message, array $context = array()) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint + public function log($level, $message, array $context = array()): void { - /** @var ConsoleLogger $consoleLogger */ - $consoleLogger = $this->getConsoleLogger(); - $message = date('m/d/y H:i:s') . ' ' . $message; - if (!$consoleLogger || !(array_key_exists($level, $this->logLevelMap))) { + if (!(array_key_exists($level, $this->logLevelMap))) { return; } /** @var string $levelKey */ $logVerbosityLevel = $this->logLevelMap[$level]; - $this->consoleLogger->log($logVerbosityLevel, $message); + $this->getConsoleLogger()->log($logVerbosityLevel, $message); } } diff --git a/src/System/Random.php b/src/System/Random.php index 2e5c1c5..b8beedd 100644 --- a/src/System/Random.php +++ b/src/System/Random.php @@ -1,28 +1,13 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 12/3/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\System; class Random { - public function getRandomString($length) + public function getRandomString(int $length): string { - return bin2hex(openssl_random_pseudo_bytes(round($length/2))); + return bin2hex(openssl_random_pseudo_bytes((int)round($length / 2))); } -} \ No newline at end of file +} diff --git a/src/System/RemoteIP.php b/src/System/RemoteIP.php index 0da88e5..b78f5db 100644 --- a/src/System/RemoteIP.php +++ b/src/System/RemoteIP.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/24/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\System; @@ -23,18 +8,18 @@ class RemoteIP { - private $ip; + private ?string $ip = null; - public function getPublicIP() + public function getPublicIP(): string { if (!$this->ip) { $client = new Client(); $response = $client->request('GET', 'https://api.ipify.org?format=json'); - $body = json_decode($response->getBody(), true); + $body = json_decode($response->getBody()->getContents(), true); $this->ip = $body['ip']; } return $this->ip; } -} \ No newline at end of file +} diff --git a/src/System/S3FilenameFormatter.php b/src/System/S3FilenameFormatter.php index b993253..e2ec74d 100644 --- a/src/System/S3FilenameFormatter.php +++ b/src/System/S3FilenameFormatter.php @@ -1,9 +1,6 @@ tag = $tag; } - public function execute(EnvironmentInterface $environment, ?string $fileKey) + public function execute(EnvironmentInterface $environment, ?string $fileKey): string { $environment = '-' . $environment->getName(); if ($this->tag->getTag()) { @@ -26,8 +23,6 @@ public function execute(EnvironmentInterface $environment, ?string $fileKey) } $replace = str_replace('{{environment}}', $environment, $fileKey); - $replace = str_replace('{{date}}', date('YmdHis'), $replace); - - return $replace; + return str_replace('{{date}}', date('YmdHis'), $replace); } } diff --git a/src/System/Tag.php b/src/System/Tag.php index a4d73f3..2843c93 100644 --- a/src/System/Tag.php +++ b/src/System/Tag.php @@ -1,9 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/25/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\System; class YamlFormatter { - /** - * Converts an array like: - * 0 => [ - * 100 => "test" - * ] - * TO: - * [ - * 100 => "test" - * ] - * @param array $input - */ - public function extractToAssociativeArray($input) + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification,SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingTraversableTypeHintSpecification + public function extractSpanList(array $input): array { - $output = array_reduce($input, function($commands, array $item) { - array_walk($item, function($values, $name) use (&$commands) { - if (isset($commands[$name]) && !is_array($commands[$name]) && is_array($values)) { - $values[] = $values; - $commands[$name] = $values; - } else if (isset($commands[$name]) && is_array($commands[$name]) && is_array($values)) { - $commands[$name] = array_merge($commands[$name], $values); - } else if (isset($commands[$name])) { // if $commands[$name] is set, turn it into an array - $commands[$name] = [ $commands[$name], $values ]; - } else { // if $commands[$name] is not set - $commands[$name] = $values; - } - }); - - return $commands; - }, []); - - return $output; - } - - public function extractSpanList($input) - { - $output = array_reduce($input, function($commands, array $item) { + return array_reduce($input, function ($commands, array $item) { if (isset($item['name'])) { $name = $item['name']; unset($item['name']); - -// if (isset($item['stages'])) { -// $item['stages'] = $this->extractToAssociativeArray($item['stages']); -// } - - if (isset($commands[$name])) { - $commands[$name] = array_merge_recursive($commands[$name], $item); - } else { - $commands[$name] = $item; - } + $commands[$name] = isset($commands[$name]) ? array_merge_recursive($commands[$name], $item) : $item; } return $commands; }, []); - - return $output; } -} \ No newline at end of file +} diff --git a/src/Tests/Unit/Commands/FactoryTest.php b/src/Tests/Unit/Commands/FactoryTest.php index a524d5e..68077fe 100644 --- a/src/Tests/Unit/Commands/FactoryTest.php +++ b/src/Tests/Unit/Commands/FactoryTest.php @@ -1,36 +1,19 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Tests\Unit\Commands; -use DI\ContainerBuilder; use Driver\Commands\Factory; -use Driver\Pipeline\Command; -use Driver\System\Configuration; -use Driver\System\DependencyConfig; +use Driver\Engines\MySql; use Driver\Tests\Unit\Helper\DI; +use PHPUnit\Framework\TestCase; -class FactoryTest extends \PHPUnit_Framework_TestCase +class FactoryTest extends TestCase { - public function testCreateReturnsPipeClass() + public function testCreateReturnsPipeClass(): void { $factory = DI::getContainer()->get(Factory::class); - $this->assertSame(Command::class, get_class($factory->create('pipeline'))); + $this->assertSame(MySql::class, get_class($factory->create('connect'))); } -} \ No newline at end of file +} diff --git a/src/Tests/Unit/Engines/MySql/CheckTest.php b/src/Tests/Unit/Engines/MySql/CheckTest.php deleted file mode 100644 index c4a2b5e..0000000 --- a/src/Tests/Unit/Engines/MySql/CheckTest.php +++ /dev/null @@ -1,39 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/19/16 - * @package default - **/ - -namespace Driver\Test\Unit\Engines\MySql; - -use Driver\Engines\MySql\Check; -use Driver\Pipeline\Master; -use Driver\Pipeline\Transport\Primary; -use Driver\Pipeline\Transport\TransportInterface; -use Driver\Tests\Unit\Helper\DI; - -class CheckTest extends \PHPUnit_Framework_TestCase -{ - protected $checkClass; - - public function testGoCorrectlyMatchesValues() - { - /** @var Check $class */ - $class = DI::getContainer()->get(Check::class); - - $this->assertTrue(is_a($class->go(new Primary(Master::class, [], [], new \Driver\System\Logs\Primary())), TransportInterface::class)); - } -} \ No newline at end of file diff --git a/src/Tests/Unit/Engines/MySql/MySqlAbstract.php b/src/Tests/Unit/Engines/MySql/MySqlAbstract.php deleted file mode 100644 index ea6a3fa..0000000 --- a/src/Tests/Unit/Engines/MySql/MySqlAbstract.php +++ /dev/null @@ -1,25 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/19/16 - * @package default - **/ - -namespace Driver\Tests\Unit\Engines\MySql; - -abstract class MySqlAbstract extends \PHPUnit_Extensions_Database_TestCase -{ - -} \ No newline at end of file diff --git a/src/Tests/Unit/Engines/MySql/Sandbox/SandboxTest.php b/src/Tests/Unit/Engines/MySql/Sandbox/SandboxTest.php index 4c12f0a..57ce3e2 100644 --- a/src/Tests/Unit/Engines/MySql/Sandbox/SandboxTest.php +++ b/src/Tests/Unit/Engines/MySql/Sandbox/SandboxTest.php @@ -1,34 +1,20 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/25/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Tests\Unit\Enines\MySql\Sandbox; use Driver\Engines\MySql\Sandbox\Sandbox; use Driver\System\AwsClientFactory; use Driver\Tests\Unit\Helper\DI; +use PHPUnit\Framework\TestCase; -class SandboxTest extends \PHPUnit_Framework_TestCase +class SandboxTest extends TestCase { - public function testGetInstanceActiveReturnsTrue() + public function testGetInstanceActiveReturnsTrue(): void { - $creator = function($serviceType, $arguments) { - $type = '\\Aws\\AwsClient'; + $creator = function ($serviceType, $arguments) { + $type = '\\Aws\\Rds\\RdsClient'; $stub = $this->getMockBuilder($type) ->setMethods(['describeDBInstances']) ->disableOriginalConstructor() @@ -47,7 +33,10 @@ public function testGetInstanceActiveReturnsTrue() return $stub; }; - $sandbox = DI::getContainer()->make(Sandbox::class, ['disableInstantiation' => true, 'awsClientFactory' => new AwsClientFactory($creator)]); + $sandbox = DI::getContainer()->make( + Sandbox::class, + ['disableInstantiation' => true, 'awsClientFactory' => new AwsClientFactory($creator)] + ); $this->assertTrue($sandbox->getInstanceActive()); } -} \ No newline at end of file +} diff --git a/src/Tests/Unit/Engines/MySql/dump.xml b/src/Tests/Unit/Engines/MySql/dump.xml deleted file mode 100644 index 217201b..0000000 --- a/src/Tests/Unit/Engines/MySql/dump.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - - - - - - - - - - 1 - test1 - value1 - - - 2 - test2 - value2 - - - 3 - test3 - value3 - - - 4 - test4 - value4 - - - 5 - test5 - value5 - - - 6 - test6 - value6 - - - 7 - test7 - value7 - - - 8 - test8 - value8 - - - 9 - test9 - value9 - - - 10 - test10 - value10 - - - - diff --git a/src/Tests/Unit/Helper/DI.php b/src/Tests/Unit/Helper/DI.php index 259c13d..fc145e8 100755 --- a/src/Tests/Unit/Helper/DI.php +++ b/src/Tests/Unit/Helper/DI.php @@ -1,35 +1,20 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Tests\Unit\Helper; +use DI\Container; use DI\ContainerBuilder; use Driver\System\DependencyConfig; class DI { - public static function getContainer() + public static function getContainer(): Container { $dependencyConfig = new DependencyConfig(); - $containerBuilder = new ContainerBuilder(); $containerBuilder->addDefinitions($dependencyConfig->get()); return $containerBuilder->build(); } -} \ No newline at end of file +} diff --git a/src/Tests/Unit/Pipeline/MasterTest.php b/src/Tests/Unit/Pipeline/MasterTest.php deleted file mode 100644 index d4e063b..0000000 --- a/src/Tests/Unit/Pipeline/MasterTest.php +++ /dev/null @@ -1,45 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/22/16 - * @package default - **/ - -namespace Driver\Tests\Unit\Pipeline; - -use Driver\Commands\Pipe; -use Driver\Pipeline\Master as PipeMaster; -use Driver\Pipeline\Span\Primary; -use Driver\Pipeline\Transport\Factory as TransportFactory; -use Driver\Pipeline\Span\Factory as PipeSpanFactory; -use Driver\System\Configuration; -use Driver\Tests\Unit\Helper\DI; - -class MasterTest extends \PHPUnit_Framework_TestCase -{ - /** @var PipeMaster $pipeMaster */ - private $pipeMaster; - - protected function setUp() - { - $this->pipeMaster = DI::getContainer()->get(PipeMaster::class); - } - - - public function testRunReturnsTransportForDefaultPipe() - { - - } -} \ No newline at end of file diff --git a/src/Tests/Unit/Pipeline/Span/FactoryTest.php b/src/Tests/Unit/Pipeline/Span/FactoryTest.php index 94a20b7..085e15e 100644 --- a/src/Tests/Unit/Pipeline/Span/FactoryTest.php +++ b/src/Tests/Unit/Pipeline/Span/FactoryTest.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Tests\Unit\Pipeline\Span; @@ -26,42 +11,35 @@ use Driver\Pipeline\Span\SpanInterface; use Driver\System\Configuration; use Driver\Tests\Unit\Helper\DI; +use PHPUnit\Framework\TestCase; -class FactoryTest extends \PHPUnit_Framework_TestCase +class FactoryTest extends TestCase { - /** @var Factory $factory */ - private $factory; + private Factory $factory; - protected function setUp() + protected function setUp(): void { $this->factory = DI::getContainer()->get(Factory::class); } - public function testCreateReturnsCorrectClass() + public function testCreateReturnsCorrectClass(): void { - $this->assertTrue(is_a($this->factory->create('default'), SpanInterface::class)); + $this->assertTrue(is_a($this->factory->create('empty'), SpanInterface::class)); } - private function runInaccessibleFunction($class, $name, $argument = null) + public function testPipelineExists(): void { - $method = new \ReflectionMethod($class, $name); - $method->setAccessible(true); - - return $method->invoke($this->factory, $argument); + $this->assertTrue( + $this->runInaccessibleFunction('pipelineExists', PipeMaster::DEFAULT_NODE) + ); } - public function testPipelineExists() + // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint,SlevomatCodingStandard.TypeHints.ReturnTypeHint + private function runInaccessibleFunction(string $name, ...$arguments) { - $this->assertTrue($this->runInaccessibleFunction($this->factory, 'pipelineExists', PipeMaster::DEFAULT_NODE)); - } - - public function testGetDefaultPipelineReturnsArray() - { - $this->assertTrue(is_array($this->runInaccessibleFunction($this->factory, 'getDefaultPipeline'))); - } + $method = new \ReflectionMethod($this->factory, $name); + $method->setAccessible(true); - public function testGetDefaultPipelineReturnsMultipleItems() - { - $this->assertGreaterThan(0, count($this->runInaccessibleFunction($this->factory, 'getDefaultPipeline'))); + return $method->invoke($this->factory, ...$arguments); } -} \ No newline at end of file +} diff --git a/src/Tests/Unit/Pipeline/Span/PrimaryTest.php b/src/Tests/Unit/Pipeline/Span/PrimaryTest.php index 29f0fe9..1b9ee08 100644 --- a/src/Tests/Unit/Pipeline/Span/PrimaryTest.php +++ b/src/Tests/Unit/Pipeline/Span/PrimaryTest.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Tests\Unit\Pipeline\Set; @@ -25,17 +10,21 @@ use Driver\System\Configuration; use Driver\Pipeline\Transport\Primary as Transport; use Driver\Tests\Unit\Helper\DI; +use PHPUnit\Framework\TestCase; -class PrimaryTest extends \PHPUnit_Framework_TestCase +class PrimaryTest extends TestCase { - public function testInvokeReturnsTransport() + public function testInvokeReturnsTransport(): void { $pipelineName = Master::DEFAULT_NODE; $configuration = new Configuration(new Configuration\FileCollector(), new Configuration\FileLoader()); - $set = DI::getContainer()->make(Primary::class, ['list' => $configuration->getNode('pipelines/' . $pipelineName)]); + $set = DI::getContainer()->make( + Primary::class, + ['list' => $configuration->getNode('pipelines/' . $pipelineName)] + ); $this->assertTrue(is_a( - $set(new Transport($pipelineName, [], [], null, new \Driver\System\Logs\Primary()), true), + $set(new Transport($pipelineName), true), TransportInterface::class )); } diff --git a/src/Tests/Unit/Pipeline/Stage/FactoryTest.php b/src/Tests/Unit/Pipeline/Stage/FactoryTest.php index 49b1b6f..daf4324 100644 --- a/src/Tests/Unit/Pipeline/Stage/FactoryTest.php +++ b/src/Tests/Unit/Pipeline/Stage/FactoryTest.php @@ -1,48 +1,25 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Tests\Unit\Pipeline\Stage; use Driver\Pipeline\Stage\Factory; use Driver\Pipeline\Stage\StageInterface; use Driver\Tests\Unit\Helper\DI; +use PHPUnit\Framework\TestCase; -class FactoryTest extends \PHPUnit_Framework_TestCase +class FactoryTest extends TestCase { - /** @var Factory $factory */ - private $factory; + private Factory $factory; - protected function setUp() + protected function setUp(): void { $this->factory = DI::getContainer()->get(Factory::class); } - public function testCreateReturnsCorrectClass() + public function testCreateReturnsCorrectClass(): void { - $this->assertTrue(is_a($this->factory->create([]), StageInterface::class)); - } - - private function runInaccessibleFunction($class, $name, $argument = null) - { - $method = new \ReflectionMethod($class, $name); - $method->setAccessible(true); - - return $method->invoke($this->factory, $argument); + $this->assertTrue(is_a($this->factory->create([], 'empty'), StageInterface::class)); } -} \ No newline at end of file +} diff --git a/src/Tests/Unit/Pipeline/Stage/PrimaryTest.php b/src/Tests/Unit/Pipeline/Stage/PrimaryTest.php index 1d90be8..8b4e05e 100644 --- a/src/Tests/Unit/Pipeline/Stage/PrimaryTest.php +++ b/src/Tests/Unit/Pipeline/Stage/PrimaryTest.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 11/5/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Tests\Unit\Pipeline\Stage; @@ -25,17 +10,21 @@ use Driver\System\Configuration; use Driver\Pipeline\Transport\Primary as Transport; use Driver\Tests\Unit\Helper\DI; +use PHPUnit\Framework\TestCase; -class PrimaryTest extends \PHPUnit_Framework_TestCase +class PrimaryTest extends TestCase { - public function testInvokeReturnsTransport() + public function testInvokeReturnsTransport(): void { $pipelineName = Master::DEFAULT_NODE; $configuration = new Configuration(new Configuration\FileCollector(), new Configuration\FileLoader()); - $set = DI::getContainer()->make(Primary::class, ['actions' => $configuration->getNode('pipelines/' . $pipelineName)]); + $set = DI::getContainer()->make( + Primary::class, + ['actions' => $configuration->getNode('pipelines/' . $pipelineName . '/0/actions'), 'name' => 'setup'] + ); $this->assertTrue(is_a( - $set(new Transport($pipelineName, [], [], null, new \Driver\System\Logs\Primary()), true), + $set(new Transport($pipelineName), true), TransportInterface::class )); } diff --git a/src/Tests/Unit/Pipeline/Transport/PrimaryTest.php b/src/Tests/Unit/Pipeline/Transport/PrimaryTest.php index eb0de7e..2d33627 100644 --- a/src/Tests/Unit/Pipeline/Transport/PrimaryTest.php +++ b/src/Tests/Unit/Pipeline/Transport/PrimaryTest.php @@ -1,21 +1,6 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/8/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Tests\Unit\Pipeline; @@ -23,22 +8,25 @@ use Driver\Pipeline\Transport\Primary; use Driver\Pipeline\Transport\Status; use Driver\Tests\Unit\Helper\DI; +use PHPUnit\Framework\TestCase; -class PrimaryTest extends \PHPUnit_Framework_TestCase +class PrimaryTest extends TestCase { - /** @var Primary */ - private $transport; - private $node = 'test'; - private $message = 'this is a message'; + private Primary $transport; + private string $node = 'test'; + private string $message = 'this is a message'; - public function setUp() + public function setUp(): void { - $this->transport = DI::getContainer()->make(Primary::class, ['pipeline' => Master::DEFAULT_NODE, 'statuses' => [], 'data' => []]); + $this->transport = DI::getContainer()->make( + Primary::class, + ['pipeline' => Master::DEFAULT_NODE, 'statuses' => [], 'data' => []] + ); parent::setUp(); } - public function testCanSetDataKey() + public function testCanSetDataKey(): void { $new = $this->transport->withNewData('sample_key', 'sample_data'); @@ -46,34 +34,41 @@ public function testCanSetDataKey() $this->assertSame('sample_data', $new->getData('sample_key')); } - public function testWithStatusReturnsNewObject() + public function testWithStatusReturnsNewObject(): void { - $this->assertTrue($this->transport !== $this->transport->withStatus(new Status($this->node, $this->message))); + $this->assertTrue( + $this->transport !== $this->transport->withStatus(new Status($this->node, $this->message)) + ); } - public function testGetStatusesByNodeReturnsValues() + public function testGetStatusesByNodeReturnsValues(): void { - $this->assertCount(1, $this->transport->withStatus(new Status($this->node, $this->message))->getStatusesByNode($this->node)); + $this->assertCount( + 1, + $this->transport->withStatus(new Status($this->node, $this->message))->getStatusesByNode($this->node) + ); $this->assertCount(0, $this->transport->getStatusesByNode($this->node)); } - public function testGetErrorsReturnsValues() + public function testGetErrorsReturnsValues(): void { - $transport = new Primary(Master::DEFAULT_NODE, [], [], new \Driver\System\Logs\Primary()); - $this->assertCount(1, $transport->withStatus(new Status($this->node, $this->message, true))->getErrors()); + $transport = new Primary(Master::DEFAULT_NODE); + $this->assertCount( + 1, + $transport->withStatus(new Status($this->node, $this->message, true))->getErrors() + ); $this->assertCount(0, $this->transport->getErrorsByNode($this->node)); } - public function testGetErrorsByNodeReturnsValues() + public function testGetErrorsByNodeReturnsValues(): void { - $this->assertCount(1, + $this->assertCount( + 1, $this->transport->withStatus(new Status($this->node, $this->message, true)) ->withStatus(new Status($this->node . '_test', $this->message)) ->withStatus(new Status($this->node, $this->message)) ->getErrors() ); - $this->assertCount(0, $this->transport->getErrors()); } - -} \ No newline at end of file +} diff --git a/src/Tests/Unit/System/ApplicationTest.php b/src/Tests/Unit/System/ApplicationTest.php index 3a5bcc1..efaf387 100644 --- a/src/Tests/Unit/System/ApplicationTest.php +++ b/src/Tests/Unit/System/ApplicationTest.php @@ -1,44 +1,25 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/8/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Tests\Unit\System; use DI\Container; -use Driver\System\Application; use Driver\Tests\Unit\Utils; -use Symfony\Component\Console\Application as ConsoleApplication; +use PHPUnit\Framework\TestCase; -class ApplicationTest extends \PHPUnit_Framework_TestCase +class ApplicationTest extends TestCase { - /** @var $container Container */ - protected $container; + protected Container $container; - protected function setUp() + protected function setUp(): void { $this->container = (new Utils())->getContainer(); } - - public function testConsoleExistsInClass() + public function testConsoleExistsInClass(): void { $application = $this->container->get('Driver\System\Application'); - $this->assertTrue(is_a($application->getConsole(), 'Symfony\Component\Console\Application')); } -} \ No newline at end of file +} diff --git a/src/Tests/Unit/System/ArgumentsTest.php b/src/Tests/Unit/System/ArgumentsTest.php deleted file mode 100755 index cc1b96c..0000000 --- a/src/Tests/Unit/System/ArgumentsTest.php +++ /dev/null @@ -1,34 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/10/16 - * @package default - **/ - -namespace Driver\Tests\Unit\System; - -use Driver\System\Arguments; - -class ArgumentsTest extends \PHPUnit_Framework_TestCase -{ - public function testDataSetAndRetrievalReturnsSameValue() - { - $value = new \stdClass(); - - $arguments = new Arguments(); - $arguments->setData('test_value', $value); - $this->assertEquals($value, $arguments->getData('test_value')); - } -} \ No newline at end of file diff --git a/src/Tests/Unit/System/ConfigurationTest.php b/src/Tests/Unit/System/ConfigurationTest.php index 364ba60..230eea8 100644 --- a/src/Tests/Unit/System/ConfigurationTest.php +++ b/src/Tests/Unit/System/ConfigurationTest.php @@ -5,8 +5,9 @@ namespace Driver\Tests\Unit\System; use Driver\System\Configuration; +use PHPUnit\Framework\TestCase; -class ConfigurationTest extends \PHPUnit_Framework_TestCase +class ConfigurationTest extends TestCase { private Configuration $configuration; @@ -18,7 +19,7 @@ public function setUp(): void public function testGetAllNodesReturnsInformation(): void { $result = $this->configuration->getNodes(); - $this->assertInternalType('array', $result); + $this->assertIsArray($result); $this->assertNotEmpty($result); } diff --git a/src/Tests/Unit/System/FactoryTest.php b/src/Tests/Unit/System/FactoryTest.php deleted file mode 100755 index a69baed..0000000 --- a/src/Tests/Unit/System/FactoryTest.php +++ /dev/null @@ -1,33 +0,0 @@ -. - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/29/16 - * @package default - **/ - -namespace Driver\Tests\Unit\System; - -use Driver\Pipeline\Transport\Error; -use Driver\System\Factory; - -class FactoryTest extends \PHPUnit_Framework_TestCase -{ - public function testFactoryReturnsObject() - { - $factory = new Factory\Base(); - $error = $factory->create(Error::class, 'Test Message'); - $this->assertEquals('Test Message', $error->getMessage()); - } -} \ No newline at end of file diff --git a/src/Tests/Unit/Utils.php b/src/Tests/Unit/Utils.php index ddf9607..bc8f407 100644 --- a/src/Tests/Unit/Utils.php +++ b/src/Tests/Unit/Utils.php @@ -1,43 +1,25 @@ . - * - * @author Joseph Maxwell - * @copyright SwiftOtter Studios, 10/8/16 - * @package default - **/ + +declare(strict_types=1); namespace Driver\Tests\Unit; +use DI\Container; use DI\ContainerBuilder; use Driver\System\DependencyConfig; class Utils { - protected $containerBuilder; + private ContainerBuilder $containerBuilder; public function __construct() { $this->containerBuilder = new ContainerBuilder(); - $this->containerBuilder->addDefinitions((new DependencyConfig())->getForTests()); + $this->containerBuilder->addDefinitions((new DependencyConfig())->get()); } - /** - * @return \DI\Container - */ - public function getContainer() + public function getContainer(): Container { return $this->containerBuilder->build(); } - -} \ No newline at end of file +} From 9285a2448f24bb5bf7e653cd3c4fe57889c865da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Thu, 6 Oct 2022 17:16:14 +0200 Subject: [PATCH 20/29] #21: Fix for config loader when `vendor/bin/driver` is a symlink --- src/System/Configuration/FolderCollectionFactory.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/System/Configuration/FolderCollectionFactory.php b/src/System/Configuration/FolderCollectionFactory.php index 657f86e..1b9aba2 100644 --- a/src/System/Configuration/FolderCollectionFactory.php +++ b/src/System/Configuration/FolderCollectionFactory.php @@ -33,12 +33,12 @@ public function create(array $allowedFolders): FolderCollection */ private function getSearchPaths(): array { - $directory = realpath($_SERVER['SCRIPT_FILENAME']); - if (strpos($directory, self::VENDOR_DIRECTORY) !== false) { - list($rootDir) = explode(self::VENDOR_DIRECTORY, $directory); + $fullPath = $_SERVER['PWD'] . '/' . $_SERVER['SCRIPT_FILENAME']; + if (strpos($fullPath, self::VENDOR_DIRECTORY) !== false) { + list($rootDir) = explode(self::VENDOR_DIRECTORY, $fullPath); return array_merge([$rootDir], $this->getVendorDirectories($rootDir)); } - return [dirname($directory, 2)]; + return [dirname($fullPath, 2)]; } /** From ec39c6e2b4cf875cfcd4487419d6a704a477111a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Thu, 17 Nov 2022 18:02:36 +0100 Subject: [PATCH 21/29] #46: Split local export mysqldump command to multiple commands (one per table) --- src/Engines/MySql/Export/CommandAssembler.php | 70 +++++++++ src/Engines/MySql/Export/Primary.php | 133 +++++------------- src/Engines/MySql/Export/TablesProvider.php | 46 ++++++ .../MySql/Export/CommandAssemblerTest.php | 99 +++++++++++++ 4 files changed, 250 insertions(+), 98 deletions(-) create mode 100644 src/Engines/MySql/Export/CommandAssembler.php create mode 100644 src/Engines/MySql/Export/TablesProvider.php create mode 100644 src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php diff --git a/src/Engines/MySql/Export/CommandAssembler.php b/src/Engines/MySql/Export/CommandAssembler.php new file mode 100644 index 0000000..56ac06d --- /dev/null +++ b/src/Engines/MySql/Export/CommandAssembler.php @@ -0,0 +1,70 @@ +tablesProvider = $tablesProvider; + } + + public function execute( + ConnectionInterface $connection, + EnvironmentInterface $environment, + string $dumpFile + ): string { + $commands = []; + $ignoredTables = $this->tablesProvider->getIgnoredTables($environment); + $emptyTables = $this->tablesProvider->getEmptyTables($environment); + foreach ($this->tablesProvider->getAllTables($connection) as $table) { + if (in_array($table, $ignoredTables) || in_array($table, $emptyTables)) { + continue; + } + $commands[] = $this->getSingleCommand($connection, [$table], $dumpFile); + } + if (!empty($emptyTables)) { + $commands[] = $this->getSingleCommand($connection, $emptyTables, $dumpFile, false); + } + if (empty($commands)) { + return ''; + } + $commands[] = "cat $dumpFile | " + . "sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > $dumpFile.gz"; + return implode(';', $commands); + } + + /** + * @param string[] $tables + */ + private function getSingleCommand( + ConnectionInterface $connection, + array $tables, + string $dumpFile, + bool $withData = true + ): string { + $parts = [ + "mysqldump --user=\"{$connection->getUser()}\"", + "--password=\"{$connection->getPassword()}\"", + "--single-transaction", + "--host={$connection->getHost()}", + $connection->getDatabase(), + implode(' ', $tables) + ]; + if (!$withData) { + $parts[] = '--no-data'; + } + $parts[] = ">> $dumpFile"; + return implode(' ', $parts); + } +} diff --git a/src/Engines/MySql/Export/Primary.php b/src/Engines/MySql/Export/Primary.php index 7de4c05..d837749 100755 --- a/src/Engines/MySql/Export/Primary.php +++ b/src/Engines/MySql/Export/Primary.php @@ -13,6 +13,8 @@ use Driver\System\Configuration; use Driver\System\Logs\LoggerInterface; use Driver\System\Random; +use Exception; +use RuntimeException; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Output\ConsoleOutput; @@ -28,6 +30,7 @@ class Primary extends Command implements CommandInterface, CleanupInterface private ?string $path = null; private Configuration $configuration; private ConsoleOutput $output; + private CommandAssembler $commandAssembler; // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( @@ -36,6 +39,7 @@ public function __construct( LoggerInterface $logger, Random $random, ConsoleOutput $output, + CommandAssembler $commandAssembler, array $properties = [] ) { $this->localConnection = $localConnection; @@ -44,6 +48,7 @@ public function __construct( $this->random = $random; $this->configuration = $configuration; $this->output = $output; + $this->commandAssembler = $commandAssembler; return parent::__construct('mysql-default-export'); } @@ -52,35 +57,39 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm $transport->getLogger()->notice("Exporting database from local MySql"); $this->output->writeln("Exporting database from local MySql"); - $transport->getLogger()->debug( - "Local connection string: " . str_replace( + try { + $command = $this->commandAssembler->execute($this->localConnection, $environment, $this->getDumpFile()); + if (empty($command)) { + throw new RuntimeException('Nothing to import'); + } + + $transport->getLogger()->debug( + "Local connection string: " . str_replace( + $this->localConnection->getPassword(), + '', + $command + ) + ); + $this->output->writeln("Local connection string: " . str_replace( $this->localConnection->getPassword(), '', - $this->assembleCommand($environment) - ) - ); - $this->output->writeln("Local connection string: " . str_replace( - $this->localConnection->getPassword(), - '', - $this->assembleCommand($environment) - )); - - $command = implode(';', array_filter([ - $this->assembleCommand($environment), - $this->assembleEmptyCommand($environment) - ])); - - $results = system($command); - - if ($results) { - $this->output->writeln('Import to RDS instance failed: ' . $results . ''); - throw new \Exception('Import to RDS instance failed: ' . $results); - } else { - $this->logger->notice("Database dump has completed."); - $this->output->writeln("Database dump has completed."); - return $transport->withStatus(new Status('sandbox_init', 'success')) - ->withNewData('dump-file', $this->getDumpFile()); + $command + )); + + $results = system($command); + + if ($results) { + throw new RuntimeException($results); + } + } catch (Exception $e) { + $this->output->writeln('Import to RDS instance failed: ' . $e->getMessage() . ''); + throw new Exception('Import to RDS instance failed: ' . $e->getMessage()); } + + $this->logger->notice("Database dump has completed."); + $this->output->writeln("Database dump has completed."); + return $transport->withStatus(new Status('sandbox_init', 'success')) + ->withNewData('dump-file', $this->getDumpFile()); } public function cleanup(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface @@ -97,78 +106,6 @@ public function getProperties(): array return $this->properties; } - public function assembleEmptyCommand(EnvironmentInterface $environment): string - { - $tables = implode(' ', $environment->getEmptyTables()); - - if (!$tables) { - return ''; - } - - return implode(' ', array_merge( - $this->getDumpCommand(), - [ - "--no-data", - $tables, - "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g'", - ">>", - $this->getDumpFile() - ] - )); - } - - public function assembleCommand(EnvironmentInterface $environment): string - { - return implode(' ', array_merge( - $this->getDumpCommand(), - [ - $this->assembleEmptyTables($environment), - $this->assembleIgnoredTables($environment), - "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g'", - ">", - $this->getDumpFile() - ] - )); - } - - /** - * @return string[] - */ - private function getDumpCommand(): array - { - return [ - "mysqldump --user=\"{$this->localConnection->getUser()}\"", - "--password=\"{$this->localConnection->getPassword()}\"", - "--single-transaction", - "--compress", - "--order-by-primary", - "--host={$this->localConnection->getHost()}", - "{$this->localConnection->getDatabase()}" - ]; - } - - private function assembleEmptyTables(EnvironmentInterface $environment): string - { - $tables = $environment->getEmptyTables(); - $output = []; - - foreach ($tables as $table) { - $output[] = '--ignore-table=' . $this->localConnection->getDatabase() . '.' . $table; - } - - return implode(' ', $output); - } - - private function assembleIgnoredTables(EnvironmentInterface $environment): string - { - $tables = $environment->getIgnoredTables(); - $output = implode(' | ', array_map(function ($table) { - return "awk '!/^INSERT INTO `{$table}` VALUES/'"; - }, $tables)); - - return $output ? ' | ' . $output : ''; - } - private function getDumpFile(): string { if (!$this->path) { diff --git a/src/Engines/MySql/Export/TablesProvider.php b/src/Engines/MySql/Export/TablesProvider.php new file mode 100644 index 0000000..9dd2de4 --- /dev/null +++ b/src/Engines/MySql/Export/TablesProvider.php @@ -0,0 +1,46 @@ +getUser()}\" --password=\"{$connection->getPassword()}\" " + . "--host=\"{$connection->getHost()}\" --skip-column-names " + . "-e \"SELECT GROUP_CONCAT(table_name SEPARATOR ',') FROM information_schema.tables " + . "WHERE table_schema = '{$connection->getDatabase()}';\""; + $result = exec($command); + if (!$result) { + throw new \RuntimeException('Unable to get table names'); + } + return explode(",", $result); + } + + /** + * @return string[] + */ + public function getEmptyTables(EnvironmentInterface $environment): array + { + return $environment->getEmptyTables(); + } + + /** + * @return string[] + */ + public function getIgnoredTables(EnvironmentInterface $environment): array + { + return $environment->getIgnoredTables(); + } +} diff --git a/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php b/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php new file mode 100644 index 0000000..d3ca759 --- /dev/null +++ b/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php @@ -0,0 +1,99 @@ +tablesProviderMock = $this->getMockBuilder(TablesProvider::class) + ->disableOriginalConstructor()->getMock(); + $this->connectionMock = $this->getMockBuilder(ConnectionInterface::class)->getMockForAbstractClass(); + $this->connectionMock->expects($this->any())->method('getUser')->willReturn('user'); + $this->connectionMock->expects($this->any())->method('getPassword')->willReturn('password'); + $this->connectionMock->expects($this->any())->method('getHost')->willReturn('host'); + $this->connectionMock->expects($this->any())->method('getDatabase')->willReturn('db'); + $this->environmentMock = $this->getMockBuilder(EnvironmentInterface::class)->getMockForAbstractClass(); + $this->commandAssembler = new CommandAssembler($this->tablesProviderMock); + } + + public function testReturnsEmptyStringIfNoTables(): void + { + $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn([]); + $this->assertSame( + '', + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + ); + } + + public function testReturnsEmptyStringIfAllTablesAreIgnored(): void + { + $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']); + $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn(['a', 'b']); + $this->assertSame( + '', + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + ); + } + + public function testReturnsCommandForNormalTables(): void + { + $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']); + $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn([]); + $this->assertSame( + 'mysqldump --user="user" --password="password" --single-transaction --host=host db a >> dump.sql;' + . 'mysqldump --user="user" --password="password" --single-transaction --host=host db b >> dump.sql;' + . "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz", + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + ); + } + + public function testReturnsCommandForEmptyTables(): void + { + $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']); + $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn([]); + $this->tablesProviderMock->expects($this->any())->method('getEmptyTables')->willReturn(['a', 'b']); + $this->assertSame( + 'mysqldump --user="user" --password="password" --single-transaction --host=host ' + . 'db a b --no-data >> dump.sql;' + . "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz", + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + ); + } + + public function testReturnsCommandForMixedTables(): void + { + $this->tablesProviderMock->expects($this->any())->method('getAllTables') + ->willReturn(['a', 'b', 'c', 'd', 'e', 'f']); + $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn(['c', 'f']); + $this->tablesProviderMock->expects($this->any())->method('getEmptyTables')->willReturn(['b', 'e']); + $this->assertSame( + 'mysqldump --user="user" --password="password" --single-transaction --host=host db a >> dump.sql;' + . 'mysqldump --user="user" --password="password" --single-transaction --host=host db d >> dump.sql;' + . 'mysqldump --user="user" --password="password" --single-transaction --host=host ' + . 'db b e --no-data >> dump.sql;' + . "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz", + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + ); + } +} From 8edbe1df3d4c50c7419502fbb0e8ba957879c12a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Thu, 17 Nov 2022 18:04:57 +0100 Subject: [PATCH 22/29] #47: Removed unnecessary transactions from transformation function --- src/Engines/MySql/Transformation.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Engines/MySql/Transformation.php b/src/Engines/MySql/Transformation.php index 19dab2d..cb0a7f3 100644 --- a/src/Engines/MySql/Transformation.php +++ b/src/Engines/MySql/Transformation.php @@ -70,14 +70,10 @@ private function applyTransformationsTo(ReconnectingPDO $connection, array $tran try { $this->logger->info("Attempting: " . $query); $this->output->writeln(" Attempting: " . $query . ''); - $connection->beginTransaction(); $connection->query($query); - $connection->commit(); - $this->logger->info("Successfully executed: " . $query); $this->output->writeln("Successfully executed: " . $query . ''); } catch (\Exception $ex) { - $connection->rollBack(); $this->logger->error("Query transformation failed: " . $query, [ $ex->getMessage(), $ex->getTraceAsString() From b6027c2a06aa0566b4fcf1b8649df74b95667aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Thu, 8 Dec 2022 18:13:56 +0100 Subject: [PATCH 23/29] Added `--no-tablespaces` to `mysqldump` commands --- src/Engines/MySql/Export/CommandAssembler.php | 1 + src/Engines/MySql/Sandbox/Export.php | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Engines/MySql/Export/CommandAssembler.php b/src/Engines/MySql/Export/CommandAssembler.php index 56ac06d..0a9b297 100644 --- a/src/Engines/MySql/Export/CommandAssembler.php +++ b/src/Engines/MySql/Export/CommandAssembler.php @@ -57,6 +57,7 @@ private function getSingleCommand( "mysqldump --user=\"{$connection->getUser()}\"", "--password=\"{$connection->getPassword()}\"", "--single-transaction", + "--no-tablespaces", "--host={$connection->getHost()}", $connection->getDatabase(), implode(' ', $tables) diff --git a/src/Engines/MySql/Sandbox/Export.php b/src/Engines/MySql/Sandbox/Export.php index af99c1f..800bc60 100755 --- a/src/Engines/MySql/Sandbox/Export.php +++ b/src/Engines/MySql/Sandbox/Export.php @@ -104,7 +104,8 @@ private function assembleCommand(string $environmentName, array $ignoredTables): "mysqldump --user={$this->connection->getUser()}", "--password={$this->connection->getPassword()}", "--host={$this->connection->getHost()}", - "--port={$this->connection->getPort()}" + "--port={$this->connection->getPort()}", + "--no-tablespaces" ], $this->getIgnoredTables($ignoredTables))); $command .= " {$this->connection->getDatabase()} "; From fe52aef1c13b271a28c70730b9d68af3bcbe191a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Wed, 21 Dec 2022 17:57:07 +0100 Subject: [PATCH 24/29] #51: Fixed foreign key checks issue after switching to per-table dump (#55) --- src/Engines/MySql/Export/CommandAssembler.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Engines/MySql/Export/CommandAssembler.php b/src/Engines/MySql/Export/CommandAssembler.php index 0a9b297..3c7bcea 100644 --- a/src/Engines/MySql/Export/CommandAssembler.php +++ b/src/Engines/MySql/Export/CommandAssembler.php @@ -7,6 +7,7 @@ use Driver\Engines\ConnectionInterface; use Driver\Pipeline\Environment\EnvironmentInterface; +use function array_unshift; use function implode; use function in_array; @@ -24,7 +25,6 @@ public function execute( EnvironmentInterface $environment, string $dumpFile ): string { - $commands = []; $ignoredTables = $this->tablesProvider->getIgnoredTables($environment); $emptyTables = $this->tablesProvider->getEmptyTables($environment); foreach ($this->tablesProvider->getAllTables($connection) as $table) { @@ -39,6 +39,12 @@ public function execute( if (empty($commands)) { return ''; } + array_unshift( + $commands, + "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'" + . ">> $dumpFile" + ); + $commands[] = "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> $dumpFile"; $commands[] = "cat $dumpFile | " . "sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > $dumpFile.gz"; return implode(';', $commands); From 8381cbc1134f054a6751c46042e92a09a1b33b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Fri, 20 Jan 2023 13:09:09 +0100 Subject: [PATCH 25/29] #56: CommandAssembler - first dump structure, then data; environments.yaml dist and example update (work in progress) --- .driver/environments.yaml.dist | 4 +- .driver/environments.yaml.example | 49 ++----------------- src/Engines/MySql/Export/CommandAssembler.php | 40 ++++++++++----- 3 files changed, 35 insertions(+), 58 deletions(-) diff --git a/.driver/environments.yaml.dist b/.driver/environments.yaml.dist index a24ee19..05c7df7 100644 --- a/.driver/environments.yaml.dist +++ b/.driver/environments.yaml.dist @@ -10,5 +10,7 @@ environments: # For example, if your `TABLE_NAME` is `core_config_data`, Driver will search for a table in the database that # ends with `core_config_data`. Thus, `core_config_data` and `sample_core_config_data` would all match. ignored_tables: # OPTIONAL, tables listed here will be ignored in the final dump with: - # mysqldump ... --ignored-tables=DATABASE.table_1 + # mysqldump ... --ignored-tables=TABLE_NAME + - TABLE_NAME + empty_tables: # OPTIONAL, tables listed here will be dumped without data - TABLE_NAME diff --git a/.driver/environments.yaml.example b/.driver/environments.yaml.example index 833d7dd..1098bc7 100644 --- a/.driver/environments.yaml.example +++ b/.driver/environments.yaml.example @@ -29,48 +29,7 @@ environments: - "UPDATE {{table_name}} SET value = 'store.local' WHERE path LIKE 'web/cookie/cookie_domain' AND scope_id = 0;" - "UPDATE {{table_name}} SET value = 'localhost' WHERE path LIKE 'catalog/search/elasticsearch%_server_hostname' AND scope_id = 0;" ignored_tables: - - setup_module - - customer_address_entity - - customer_address_entity_datetime - - customer_address_entity_decimal - - customer_address_entity_int - - customer_address_entity_text - - customer_address_entity_varchar - - customer_entity - - customer_entity_datetime - - customer_entity_decimal - - customer_entity_int - - customer_entity_text - - customer_entity_varchar - - sales_creditmemo - - sales_credimemo_comment - - sales_creditmemo_grid - - sales_creditmemo_item - - sales_invoice - - sales_invoice_comment - - sales_invoice_grid - - sales_invoice_item - - sales_order - - sales_order_address - - sales_order_grid - - sales_order_item - - sales_order_payment - - sales_order_status_history - - sales_shipment - - sales_shipment_comment - - sales_shipment_grid - - sales_shipment_item - - sales_shipment_track - - sales_invoiced_aggregated - - sales_invoiced_aggregated_order - - sales_payment_transaction - - sales_order_aggregated_created - - sales_order_tax - - sales_order_tax_item - - sales_quote - - sales_quote_address - - sales_quote_address_item - - sales_quote_item - - sales_quote_item_option - - sales_quote_payment - - sales_quote_shipping_rate + - some_non_magento_table + empty_tables: + - customer_log + - customer_visitor diff --git a/src/Engines/MySql/Export/CommandAssembler.php b/src/Engines/MySql/Export/CommandAssembler.php index 3c7bcea..7044d17 100644 --- a/src/Engines/MySql/Export/CommandAssembler.php +++ b/src/Engines/MySql/Export/CommandAssembler.php @@ -27,14 +27,12 @@ public function execute( ): string { $ignoredTables = $this->tablesProvider->getIgnoredTables($environment); $emptyTables = $this->tablesProvider->getEmptyTables($environment); + $commands = [$this->getStructureCommand($connection, $ignoredTables, $dumpFile)]; foreach ($this->tablesProvider->getAllTables($connection) as $table) { if (in_array($table, $ignoredTables) || in_array($table, $emptyTables)) { continue; } - $commands[] = $this->getSingleCommand($connection, [$table], $dumpFile); - } - if (!empty($emptyTables)) { - $commands[] = $this->getSingleCommand($connection, $emptyTables, $dumpFile, false); + $commands[] = $this->getDataCommand($connection, [$table], $dumpFile); } if (empty($commands)) { return ''; @@ -51,26 +49,44 @@ public function execute( } /** - * @param string[] $tables + * @param string[] $ignoredTables */ - private function getSingleCommand( + private function getStructureCommand( ConnectionInterface $connection, - array $tables, - string $dumpFile, - bool $withData = true + array $ignoredTables, + string $dumpFile ): string { $parts = [ "mysqldump --user=\"{$connection->getUser()}\"", "--password=\"{$connection->getPassword()}\"", "--single-transaction", "--no-tablespaces", + "--no-data", + "--host={$connection->getHost()}", + $connection->getDatabase() + ]; + foreach ($ignoredTables as $table) { + $parts[] = "--ignore-table={$connection->getDatabase()}.{$table}"; + } + $parts[] = ">> $dumpFile"; + return implode(' ', $parts); + } + + /** + * @param string[] $tables + */ + private function getDataCommand(ConnectionInterface $connection, array $tables, string $dumpFile): string + { + $parts = [ + "mysqldump --user=\"{$connection->getUser()}\"", + "--password=\"{$connection->getPassword()}\"", + "--single-transaction", + "--no-tablespaces", + "--no-create-info", "--host={$connection->getHost()}", $connection->getDatabase(), implode(' ', $tables) ]; - if (!$withData) { - $parts[] = '--no-data'; - } $parts[] = ">> $dumpFile"; return implode(' ', $parts); } From ae343c648b0df2d07848890ae7a09946e318085b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Fri, 20 Jan 2023 16:50:38 +0100 Subject: [PATCH 26/29] #57: Fix for "Unable to fork" issue when executing a lot of mysqldump commands --- src/Engines/MySql/Export/CommandAssembler.php | 9 ++-- src/Engines/MySql/Export/Primary.php | 37 ++++++------- .../MySql/Export/CommandAssemblerTest.php | 52 ++++++++++++------- 3 files changed, 55 insertions(+), 43 deletions(-) diff --git a/src/Engines/MySql/Export/CommandAssembler.php b/src/Engines/MySql/Export/CommandAssembler.php index 3c7bcea..dcba6db 100644 --- a/src/Engines/MySql/Export/CommandAssembler.php +++ b/src/Engines/MySql/Export/CommandAssembler.php @@ -20,11 +20,14 @@ public function __construct(TablesProvider $tablesProvider) $this->tablesProvider = $tablesProvider; } + /** + * @return string[] + */ public function execute( ConnectionInterface $connection, EnvironmentInterface $environment, string $dumpFile - ): string { + ): array { $ignoredTables = $this->tablesProvider->getIgnoredTables($environment); $emptyTables = $this->tablesProvider->getEmptyTables($environment); foreach ($this->tablesProvider->getAllTables($connection) as $table) { @@ -37,7 +40,7 @@ public function execute( $commands[] = $this->getSingleCommand($connection, $emptyTables, $dumpFile, false); } if (empty($commands)) { - return ''; + return []; } array_unshift( $commands, @@ -47,7 +50,7 @@ public function execute( $commands[] = "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> $dumpFile"; $commands[] = "cat $dumpFile | " . "sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > $dumpFile.gz"; - return implode(';', $commands); + return $commands; } /** diff --git a/src/Engines/MySql/Export/Primary.php b/src/Engines/MySql/Export/Primary.php index d837749..0c35fff 100755 --- a/src/Engines/MySql/Export/Primary.php +++ b/src/Engines/MySql/Export/Primary.php @@ -13,10 +13,10 @@ use Driver\System\Configuration; use Driver\System\Logs\LoggerInterface; use Driver\System\Random; -use Exception; use RuntimeException; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Output\ConsoleOutput; +use Throwable; class Primary extends Command implements CommandInterface, CleanupInterface { @@ -58,32 +58,25 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm $this->output->writeln("Exporting database from local MySql"); try { - $command = $this->commandAssembler->execute($this->localConnection, $environment, $this->getDumpFile()); - if (empty($command)) { + $commands = $this->commandAssembler->execute($this->localConnection, $environment, $this->getDumpFile()); + if (empty($commands)) { throw new RuntimeException('Nothing to import'); } - $transport->getLogger()->debug( - "Local connection string: " . str_replace( - $this->localConnection->getPassword(), - '', - $command - ) - ); - $this->output->writeln("Local connection string: " . str_replace( - $this->localConnection->getPassword(), - '', - $command - )); - - $results = system($command); - - if ($results) { - throw new RuntimeException($results); + foreach ($commands as $command) { + $strippedCommand = str_replace($this->localConnection->getPassword(), '', $command); + $transport->getLogger()->debug('Command: ' . $strippedCommand); + $resultCode = 0; + $result = system($command, $resultCode); + if ($result === false || $resultCode !== 0) { + $message = sprintf('Error (%s) when executing command: %s', $resultCode, $strippedCommand); + $this->output->writeln("${$message}"); + throw new RuntimeException($message); + } } - } catch (Exception $e) { + } catch (Throwable $e) { $this->output->writeln('Import to RDS instance failed: ' . $e->getMessage() . ''); - throw new Exception('Import to RDS instance failed: ' . $e->getMessage()); + throw new RuntimeException('Import to RDS instance failed: ' . $e->getMessage()); } $this->logger->notice("Database dump has completed."); diff --git a/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php b/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php index d3ca759..9b7a8fe 100644 --- a/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php +++ b/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php @@ -37,62 +37,78 @@ public function setUp(): void $this->commandAssembler = new CommandAssembler($this->tablesProviderMock); } - public function testReturnsEmptyStringIfNoTables(): void + public function testReturnsEmptyArrayIfNoTables(): void { $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn([]); $this->assertSame( - '', + [], $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') ); } - public function testReturnsEmptyStringIfAllTablesAreIgnored(): void + public function testReturnsEmptyArrayIfAllTablesAreIgnored(): void { $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']); $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn(['a', 'b']); $this->assertSame( - '', + [], $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') ); } - public function testReturnsCommandForNormalTables(): void + public function testReturnsCommandsForNormalTables(): void { $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']); $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn([]); $this->assertSame( - 'mysqldump --user="user" --password="password" --single-transaction --host=host db a >> dump.sql;' - . 'mysqldump --user="user" --password="password" --single-transaction --host=host db b >> dump.sql;' - . "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz", + [ + "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql", + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' + . 'db a >> dump.sql', + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' + . 'db b >> dump.sql', + "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql", + "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz" + ], $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') ); } - public function testReturnsCommandForEmptyTables(): void + public function testReturnsCommandsForEmptyTables(): void { $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']); $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn([]); $this->tablesProviderMock->expects($this->any())->method('getEmptyTables')->willReturn(['a', 'b']); $this->assertSame( - 'mysqldump --user="user" --password="password" --single-transaction --host=host ' - . 'db a b --no-data >> dump.sql;' - . "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz", + [ + "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql", + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' + . 'db a b --no-data >> dump.sql', + "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql", + "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz" + ], $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') ); } - public function testReturnsCommandForMixedTables(): void + public function testReturnsCommandsForMixedTables(): void { $this->tablesProviderMock->expects($this->any())->method('getAllTables') ->willReturn(['a', 'b', 'c', 'd', 'e', 'f']); $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn(['c', 'f']); $this->tablesProviderMock->expects($this->any())->method('getEmptyTables')->willReturn(['b', 'e']); $this->assertSame( - 'mysqldump --user="user" --password="password" --single-transaction --host=host db a >> dump.sql;' - . 'mysqldump --user="user" --password="password" --single-transaction --host=host db d >> dump.sql;' - . 'mysqldump --user="user" --password="password" --single-transaction --host=host ' - . 'db b e --no-data >> dump.sql;' - . "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz", + [ + "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql", + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' + . 'db a >> dump.sql', + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' + . 'db d >> dump.sql', + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' + . 'db b e --no-data >> dump.sql', + "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql", + "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz" + ], $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') ); } From 4d37958c520ced7cb1cc8837392384a4577d5d2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Tue, 24 Jan 2023 18:03:01 +0100 Subject: [PATCH 27/29] #60: Fix for incorrect caching mechanism of getFilename method (#63) --- src/Engines/MySql/Sandbox/Export.php | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Engines/MySql/Sandbox/Export.php b/src/Engines/MySql/Sandbox/Export.php index 800bc60..f3ad9ee 100755 --- a/src/Engines/MySql/Sandbox/Export.php +++ b/src/Engines/MySql/Sandbox/Export.php @@ -15,12 +15,15 @@ use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Output\ConsoleOutput; +use function array_key_exists; + class Export extends Command implements CommandInterface, CleanupInterface { private RemoteConnectionInterface $connection; private Ssl $ssl; private Random $random; - private ?string $filename = null; + /** @var array */ + private array $filenames = []; private Configuration $configuration; // phpcs:ignore SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingTraversableTypeHintSpecification private array $properties; @@ -148,8 +151,8 @@ private function compressOutput(): bool private function getFilename(string $environmentName): string { - if ($this->filename) { - return $this->filename; + if (array_key_exists($environmentName, $this->filenames)) { + return $this->filenames[$environmentName]; } $path = $this->configuration->getNode('connections/rds/dump-path'); @@ -161,7 +164,7 @@ private function getFilename(string $environmentName): string . ($this->compressOutput() ? '.gz' : '.sql'); } while (file_exists($file)); - $this->filename = $file; - return $this->filename; + $this->filenames[$environmentName] = $file; + return $this->filenames[$environmentName]; } } From 962fbf7b152cdec0162eabef7d32dcb9a87ddbdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Wed, 25 Jan 2023 17:29:18 +0100 Subject: [PATCH 28/29] #61: Workaround for AWS RDS not allowing creating triggers --- src/Engines/MySql/Export/CommandAssembler.php | 50 +++++++++++++++---- src/Engines/MySql/Export/Primary.php | 29 ++++++----- src/Engines/MySql/Sandbox/Export.php | 23 ++++----- src/Engines/MySql/Sandbox/Import.php | 13 +++-- src/Engines/MySql/Transformation.php | 5 +- .../MySql/Export/CommandAssemblerTest.php | 47 ++++++++--------- 6 files changed, 96 insertions(+), 71 deletions(-) diff --git a/src/Engines/MySql/Export/CommandAssembler.php b/src/Engines/MySql/Export/CommandAssembler.php index e0d58ea..dd0f0c1 100644 --- a/src/Engines/MySql/Export/CommandAssembler.php +++ b/src/Engines/MySql/Export/CommandAssembler.php @@ -7,6 +7,7 @@ use Driver\Engines\ConnectionInterface; use Driver\Pipeline\Environment\EnvironmentInterface; +use function array_diff; use function array_unshift; use function implode; use function in_array; @@ -26,28 +27,29 @@ public function __construct(TablesProvider $tablesProvider) public function execute( ConnectionInterface $connection, EnvironmentInterface $environment, - string $dumpFile + string $dumpFile, + string $triggersDumpFile ): array { + $allTables = $this->tablesProvider->getAllTables($connection); $ignoredTables = $this->tablesProvider->getIgnoredTables($environment); + if (array_diff($allTables, $ignoredTables) === []) { + return []; + } $emptyTables = $this->tablesProvider->getEmptyTables($environment); $commands = [$this->getStructureCommand($connection, $ignoredTables, $dumpFile)]; - foreach ($this->tablesProvider->getAllTables($connection) as $table) { + foreach ($allTables as $table) { if (in_array($table, $ignoredTables) || in_array($table, $emptyTables)) { continue; } $commands[] = $this->getDataCommand($connection, [$table], $dumpFile); } - if (empty($commands)) { - return []; - } array_unshift( $commands, "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'" - . ">> $dumpFile" + . " | gzip >> $dumpFile" ); - $commands[] = "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> $dumpFile"; - $commands[] = "cat $dumpFile | " - . "sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > $dumpFile.gz"; + $commands[] = "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' | gzip >> $dumpFile"; + $commands[] = $this->getTriggersCommand($connection, $ignoredTables, $triggersDumpFile); return $commands; } @@ -65,12 +67,15 @@ private function getStructureCommand( "--single-transaction", "--no-tablespaces", "--no-data", + "--skip-triggers", "--host={$connection->getHost()}", $connection->getDatabase() ]; foreach ($ignoredTables as $table) { $parts[] = "--ignore-table={$connection->getDatabase()}.{$table}"; } + $parts[] = "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g'"; + $parts[] = "| gzip"; $parts[] = ">> $dumpFile"; return implode(' ', $parts); } @@ -86,10 +91,37 @@ private function getDataCommand(ConnectionInterface $connection, array $tables, "--single-transaction", "--no-tablespaces", "--no-create-info", + "--skip-triggers", "--host={$connection->getHost()}", $connection->getDatabase(), implode(' ', $tables) ]; + $parts[] = "| gzip"; + $parts[] = ">> $dumpFile"; + return implode(' ', $parts); + } + + /** + * @param string[] $ignoredTables + */ + private function getTriggersCommand(ConnectionInterface $connection, array $ignoredTables, string $dumpFile): string + { + $parts = [ + "mysqldump --user=\"{$connection->getUser()}\"", + "--password=\"{$connection->getPassword()}\"", + "--single-transaction", + "--no-tablespaces", + "--no-data", + "--no-create-info", + "--triggers", + "--host={$connection->getHost()}", + $connection->getDatabase() + ]; + foreach ($ignoredTables as $table) { + $parts[] = "--ignore-table={$connection->getDatabase()}.{$table}"; + } + $parts[] = "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g'"; + $parts[] = "| gzip"; $parts[] = ">> $dumpFile"; return implode(' ', $parts); } diff --git a/src/Engines/MySql/Export/Primary.php b/src/Engines/MySql/Export/Primary.php index 0c35fff..3444675 100755 --- a/src/Engines/MySql/Export/Primary.php +++ b/src/Engines/MySql/Export/Primary.php @@ -27,7 +27,8 @@ class Primary extends Command implements CommandInterface, CleanupInterface private array $properties; private LoggerInterface $logger; private Random $random; - private ?string $path = null; + /** @var array */ + private array $filePaths = []; private Configuration $configuration; private ConsoleOutput $output; private CommandAssembler $commandAssembler; @@ -58,7 +59,8 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm $this->output->writeln("Exporting database from local MySql"); try { - $commands = $this->commandAssembler->execute($this->localConnection, $environment, $this->getDumpFile()); + $commands = $this->commandAssembler + ->execute($this->localConnection, $environment, $this->getDumpFile(), $this->getDumpFile('triggers')); if (empty($commands)) { throw new RuntimeException('Nothing to import'); } @@ -70,7 +72,7 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm $result = system($command, $resultCode); if ($result === false || $resultCode !== 0) { $message = sprintf('Error (%s) when executing command: %s', $resultCode, $strippedCommand); - $this->output->writeln("${$message}"); + $this->output->writeln("$message"); throw new RuntimeException($message); } } @@ -82,14 +84,17 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm $this->logger->notice("Database dump has completed."); $this->output->writeln("Database dump has completed."); return $transport->withStatus(new Status('sandbox_init', 'success')) - ->withNewData('dump-file', $this->getDumpFile()); + ->withNewData('dump-file', $this->getDumpFile()) + ->withNewData('triggers-dump-file', $this->getDumpFile('triggers')); } public function cleanup(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { - if ($this->getDumpFile() && file_exists($this->getDumpFile())) { - @unlink($this->getDumpFile()); - } + array_walk($this->filePaths, function (string $filePath): void { + if ($filePath && file_exists($filePath)) { + @unlink($filePath); + } + }); return $transport; } @@ -99,18 +104,18 @@ public function getProperties(): array return $this->properties; } - private function getDumpFile(): string + private function getDumpFile(string $code = 'default'): string { - if (!$this->path) { + if (!\array_key_exists($code, $this->filePaths)) { $path = $this->configuration->getNode('connections/mysql/dump-path'); if (!$path) { $path = self::DEFAULT_DUMP_PATH; } - $filename = 'driver-' . $this->random->getRandomString(6) . '.sql'; + $filename = 'driver-' . $this->random->getRandomString(6) . '.gz'; - $this->path = $path . '/' . $filename; + $this->filePaths[$code] = $path . '/' . $filename; } - return $this->path; + return $this->filePaths[$code]; } } diff --git a/src/Engines/MySql/Sandbox/Export.php b/src/Engines/MySql/Sandbox/Export.php index 800bc60..29098f4 100755 --- a/src/Engines/MySql/Sandbox/Export.php +++ b/src/Engines/MySql/Sandbox/Export.php @@ -68,7 +68,11 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm ); $environmentName = $environment->getName(); - $command = $this->assembleCommand($environmentName, $environment->getIgnoredTables()); + $command = $this->assembleCommand( + $environmentName, + $environment->getIgnoredTables(), + $transport->getData('triggers-dump-file') + ); $this->files[] = $this->getFilename($environmentName); @@ -98,8 +102,9 @@ public function cleanup(TransportInterface $transport, EnvironmentInterface $env /** * @param string[] $ignoredTables */ - private function assembleCommand(string $environmentName, array $ignoredTables): string + private function assembleCommand(string $environmentName, array $ignoredTables, string $triggersDumpFile): string { + $filename = $this->getFilename($environmentName); $command = implode(' ', array_merge([ "mysqldump --user={$this->connection->getUser()}", "--password={$this->connection->getPassword()}", @@ -107,21 +112,13 @@ private function assembleCommand(string $environmentName, array $ignoredTables): "--port={$this->connection->getPort()}", "--no-tablespaces" ], $this->getIgnoredTables($ignoredTables))); - $command .= " {$this->connection->getDatabase()} "; $command .= "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' "; - if ($this->compressOutput()) { - $command .= ' ' . implode(' ', [ - '|', - 'gzip --best' - ]); + $command .= "| gzip --best "; } - - $command .= ' ' . implode(' ', [ - '>', - $this->getFilename($environmentName) - ]); + $command .= "> $filename;"; + $command .= ($this->compressOutput() ? "cat" : "gunzip < ") . " $triggersDumpFile >> $filename;"; return $command; } diff --git a/src/Engines/MySql/Sandbox/Import.php b/src/Engines/MySql/Sandbox/Import.php index 771489a..698cfaf 100755 --- a/src/Engines/MySql/Sandbox/Import.php +++ b/src/Engines/MySql/Sandbox/Import.php @@ -74,14 +74,13 @@ public function getProperties(): array public function assembleCommand(string $path): string { $command = implode(' ', [ + "gunzip < $path |", "mysql --user={$this->remoteConnection->getUser()}", - "--password={$this->remoteConnection->getPassword()}", - "--host={$this->remoteConnection->getHost()}", - "--port={$this->remoteConnection->getPort()}", - $this->remoteConnection->useSsl() ? "--ssl-ca={$this->ssl->getPath()}" : "", - "{$this->remoteConnection->getDatabase()}", - "<", - $path + "--password={$this->remoteConnection->getPassword()}", + "--host={$this->remoteConnection->getHost()}", + "--port={$this->remoteConnection->getPort()}", + $this->remoteConnection->useSsl() ? "--ssl-ca={$this->ssl->getPath()}" : "", + "{$this->remoteConnection->getDatabase()}" ]); if ( diff --git a/src/Engines/MySql/Transformation.php b/src/Engines/MySql/Transformation.php index cb0a7f3..a5427d1 100644 --- a/src/Engines/MySql/Transformation.php +++ b/src/Engines/MySql/Transformation.php @@ -78,10 +78,7 @@ private function applyTransformationsTo(ReconnectingPDO $connection, array $tran $ex->getMessage(), $ex->getTraceAsString() ]); - $this->output->writeln("Query transformation failed: " . $query, [ - $ex->getMessage(), - $ex->getTraceAsString() - ] . ''); + $this->output->writeln("Query transformation failed: " . $query . ''); } }); } diff --git a/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php b/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php index 9b7a8fe..595df08 100644 --- a/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php +++ b/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php @@ -42,7 +42,7 @@ public function testReturnsEmptyArrayIfNoTables(): void $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn([]); $this->assertSame( [], - $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.gz', 'triggers.gz') ); } @@ -52,7 +52,7 @@ public function testReturnsEmptyArrayIfAllTablesAreIgnored(): void $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn(['a', 'b']); $this->assertSame( [], - $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.gz', 'triggers.gz') ); } @@ -62,15 +62,14 @@ public function testReturnsCommandsForNormalTables(): void $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn([]); $this->assertSame( [ - "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql", - 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' - . 'db a >> dump.sql', - 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' - . 'db b >> dump.sql', - "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql", - "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz" + "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;' | gzip >> dump.gz", + "mysqldump --user=\"user\" --password=\"password\" --single-transaction --no-tablespaces --no-data --skip-triggers --host=host db | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip >> dump.gz", + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --no-create-info --skip-triggers --host=host db a | gzip >> dump.gz', + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --no-create-info --skip-triggers --host=host db b | gzip >> dump.gz', + "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' | gzip >> dump.gz", + "mysqldump --user=\"user\" --password=\"password\" --single-transaction --no-tablespaces --no-data --no-create-info --triggers --host=host db | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip >> triggers.gz" ], - $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.gz', 'triggers.gz') ); } @@ -81,13 +80,12 @@ public function testReturnsCommandsForEmptyTables(): void $this->tablesProviderMock->expects($this->any())->method('getEmptyTables')->willReturn(['a', 'b']); $this->assertSame( [ - "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql", - 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' - . 'db a b --no-data >> dump.sql', - "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql", - "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz" + "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;' | gzip >> dump.gz", + "mysqldump --user=\"user\" --password=\"password\" --single-transaction --no-tablespaces --no-data --skip-triggers --host=host db | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip >> dump.gz", + "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' | gzip >> dump.gz", + "mysqldump --user=\"user\" --password=\"password\" --single-transaction --no-tablespaces --no-data --no-create-info --triggers --host=host db | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip >> triggers.gz" ], - $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.gz', 'triggers.gz') ); } @@ -99,17 +97,14 @@ public function testReturnsCommandsForMixedTables(): void $this->tablesProviderMock->expects($this->any())->method('getEmptyTables')->willReturn(['b', 'e']); $this->assertSame( [ - "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql", - 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' - . 'db a >> dump.sql', - 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' - . 'db d >> dump.sql', - 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' - . 'db b e --no-data >> dump.sql', - "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql", - "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz" + "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;' | gzip >> dump.gz", + "mysqldump --user=\"user\" --password=\"password\" --single-transaction --no-tablespaces --no-data --skip-triggers --host=host db --ignore-table=db.c --ignore-table=db.f | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip >> dump.gz", + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --no-create-info --skip-triggers --host=host db a | gzip >> dump.gz', + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --no-create-info --skip-triggers --host=host db d | gzip >> dump.gz', + "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' | gzip >> dump.gz", + "mysqldump --user=\"user\" --password=\"password\" --single-transaction --no-tablespaces --no-data --no-create-info --triggers --host=host db --ignore-table=db.c --ignore-table=db.f | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip >> triggers.gz" ], - $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.gz', 'triggers.gz') ); } } From 093f610f92185156492d8e560a69abdce65cfe77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Tue, 31 Jan 2023 11:10:54 +0100 Subject: [PATCH 29/29] #65: getAllTables GROUP_CONCAT issue fixed (#66) --- src/Engines/MySql/Export/TablesProvider.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Engines/MySql/Export/TablesProvider.php b/src/Engines/MySql/Export/TablesProvider.php index 9dd2de4..5346f6e 100644 --- a/src/Engines/MySql/Export/TablesProvider.php +++ b/src/Engines/MySql/Export/TablesProvider.php @@ -19,8 +19,8 @@ public function getAllTables(ConnectionInterface $connection): array { $command = "mysql --user=\"{$connection->getUser()}\" --password=\"{$connection->getPassword()}\" " . "--host=\"{$connection->getHost()}\" --skip-column-names " - . "-e \"SELECT GROUP_CONCAT(table_name SEPARATOR ',') FROM information_schema.tables " - . "WHERE table_schema = '{$connection->getDatabase()}';\""; + . "-e \"SET SESSION group_concat_max_len = 1000000; SELECT GROUP_CONCAT(table_name SEPARATOR ',') " + . " FROM information_schema.tables WHERE table_schema = '{$connection->getDatabase()}';\""; $result = exec($command); if (!$result) { throw new \RuntimeException('Unable to get table names');