From ae343c648b0df2d07848890ae7a09946e318085b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Fri, 20 Jan 2023 16:50:38 +0100 Subject: [PATCH] #57: Fix for "Unable to fork" issue when executing a lot of mysqldump commands --- src/Engines/MySql/Export/CommandAssembler.php | 9 ++-- src/Engines/MySql/Export/Primary.php | 37 ++++++------- .../MySql/Export/CommandAssemblerTest.php | 52 ++++++++++++------- 3 files changed, 55 insertions(+), 43 deletions(-) diff --git a/src/Engines/MySql/Export/CommandAssembler.php b/src/Engines/MySql/Export/CommandAssembler.php index 3c7bcea..dcba6db 100644 --- a/src/Engines/MySql/Export/CommandAssembler.php +++ b/src/Engines/MySql/Export/CommandAssembler.php @@ -20,11 +20,14 @@ public function __construct(TablesProvider $tablesProvider) $this->tablesProvider = $tablesProvider; } + /** + * @return string[] + */ public function execute( ConnectionInterface $connection, EnvironmentInterface $environment, string $dumpFile - ): string { + ): array { $ignoredTables = $this->tablesProvider->getIgnoredTables($environment); $emptyTables = $this->tablesProvider->getEmptyTables($environment); foreach ($this->tablesProvider->getAllTables($connection) as $table) { @@ -37,7 +40,7 @@ public function execute( $commands[] = $this->getSingleCommand($connection, $emptyTables, $dumpFile, false); } if (empty($commands)) { - return ''; + return []; } array_unshift( $commands, @@ -47,7 +50,7 @@ public function execute( $commands[] = "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> $dumpFile"; $commands[] = "cat $dumpFile | " . "sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > $dumpFile.gz"; - return implode(';', $commands); + return $commands; } /** diff --git a/src/Engines/MySql/Export/Primary.php b/src/Engines/MySql/Export/Primary.php index d837749..0c35fff 100755 --- a/src/Engines/MySql/Export/Primary.php +++ b/src/Engines/MySql/Export/Primary.php @@ -13,10 +13,10 @@ use Driver\System\Configuration; use Driver\System\Logs\LoggerInterface; use Driver\System\Random; -use Exception; use RuntimeException; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Output\ConsoleOutput; +use Throwable; class Primary extends Command implements CommandInterface, CleanupInterface { @@ -58,32 +58,25 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm $this->output->writeln("Exporting database from local MySql"); try { - $command = $this->commandAssembler->execute($this->localConnection, $environment, $this->getDumpFile()); - if (empty($command)) { + $commands = $this->commandAssembler->execute($this->localConnection, $environment, $this->getDumpFile()); + if (empty($commands)) { throw new RuntimeException('Nothing to import'); } - $transport->getLogger()->debug( - "Local connection string: " . str_replace( - $this->localConnection->getPassword(), - '', - $command - ) - ); - $this->output->writeln("Local connection string: " . str_replace( - $this->localConnection->getPassword(), - '', - $command - )); - - $results = system($command); - - if ($results) { - throw new RuntimeException($results); + foreach ($commands as $command) { + $strippedCommand = str_replace($this->localConnection->getPassword(), '', $command); + $transport->getLogger()->debug('Command: ' . $strippedCommand); + $resultCode = 0; + $result = system($command, $resultCode); + if ($result === false || $resultCode !== 0) { + $message = sprintf('Error (%s) when executing command: %s', $resultCode, $strippedCommand); + $this->output->writeln("${$message}"); + throw new RuntimeException($message); + } } - } catch (Exception $e) { + } catch (Throwable $e) { $this->output->writeln('Import to RDS instance failed: ' . $e->getMessage() . ''); - throw new Exception('Import to RDS instance failed: ' . $e->getMessage()); + throw new RuntimeException('Import to RDS instance failed: ' . $e->getMessage()); } $this->logger->notice("Database dump has completed."); diff --git a/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php b/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php index d3ca759..9b7a8fe 100644 --- a/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php +++ b/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php @@ -37,62 +37,78 @@ public function setUp(): void $this->commandAssembler = new CommandAssembler($this->tablesProviderMock); } - public function testReturnsEmptyStringIfNoTables(): void + public function testReturnsEmptyArrayIfNoTables(): void { $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn([]); $this->assertSame( - '', + [], $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') ); } - public function testReturnsEmptyStringIfAllTablesAreIgnored(): void + public function testReturnsEmptyArrayIfAllTablesAreIgnored(): void { $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']); $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn(['a', 'b']); $this->assertSame( - '', + [], $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') ); } - public function testReturnsCommandForNormalTables(): void + public function testReturnsCommandsForNormalTables(): void { $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']); $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn([]); $this->assertSame( - 'mysqldump --user="user" --password="password" --single-transaction --host=host db a >> dump.sql;' - . 'mysqldump --user="user" --password="password" --single-transaction --host=host db b >> dump.sql;' - . "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz", + [ + "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql", + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' + . 'db a >> dump.sql', + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' + . 'db b >> dump.sql', + "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql", + "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz" + ], $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') ); } - public function testReturnsCommandForEmptyTables(): void + public function testReturnsCommandsForEmptyTables(): void { $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']); $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn([]); $this->tablesProviderMock->expects($this->any())->method('getEmptyTables')->willReturn(['a', 'b']); $this->assertSame( - 'mysqldump --user="user" --password="password" --single-transaction --host=host ' - . 'db a b --no-data >> dump.sql;' - . "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz", + [ + "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql", + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' + . 'db a b --no-data >> dump.sql', + "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql", + "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz" + ], $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') ); } - public function testReturnsCommandForMixedTables(): void + public function testReturnsCommandsForMixedTables(): void { $this->tablesProviderMock->expects($this->any())->method('getAllTables') ->willReturn(['a', 'b', 'c', 'd', 'e', 'f']); $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn(['c', 'f']); $this->tablesProviderMock->expects($this->any())->method('getEmptyTables')->willReturn(['b', 'e']); $this->assertSame( - 'mysqldump --user="user" --password="password" --single-transaction --host=host db a >> dump.sql;' - . 'mysqldump --user="user" --password="password" --single-transaction --host=host db d >> dump.sql;' - . 'mysqldump --user="user" --password="password" --single-transaction --host=host ' - . 'db b e --no-data >> dump.sql;' - . "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz", + [ + "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql", + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' + . 'db a >> dump.sql', + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' + . 'db d >> dump.sql', + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' + . 'db b e --no-data >> dump.sql', + "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql", + "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz" + ], $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') ); }