Skip to content

Commit

Permalink
#57: Fix for "Unable to fork" issue when executing a lot of mysqldump…
Browse files Browse the repository at this point in the history 
… commands
  • Loading branch information
@michalbiarda
michalbiarda committed Jan 20, 2023
1 parent fe52aef commit ae343c6
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 43 deletions.
9 changes: 6 additions & 3 deletions src/Engines/MySql/Export/CommandAssembler.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,14 @@ public function __construct(TablesProvider $tablesProvider)
$this->tablesProvider = $tablesProvider;
}

/**
* @return string[]
*/
public function execute(
ConnectionInterface $connection,
EnvironmentInterface $environment,
string $dumpFile
): string {
): array {
$ignoredTables = $this->tablesProvider->getIgnoredTables($environment);
$emptyTables = $this->tablesProvider->getEmptyTables($environment);
foreach ($this->tablesProvider->getAllTables($connection) as $table) {
Expand All @@ -37,7 +40,7 @@ public function execute(
$commands[] = $this->getSingleCommand($connection, $emptyTables, $dumpFile, false);
}
if (empty($commands)) {
return '';
return [];
}
array_unshift(
$commands,
Expand All @@ -47,7 +50,7 @@ public function execute(
$commands[] = "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> $dumpFile";
$commands[] = "cat $dumpFile | "
. "sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > $dumpFile.gz";
return implode(';', $commands);
return $commands;
}

/**
Expand Down
37 changes: 15 additions & 22 deletions src/Engines/MySql/Export/Primary.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@
use Driver\System\Configuration;
use Driver\System\Logs\LoggerInterface;
use Driver\System\Random;
use Exception;
use RuntimeException;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Output\ConsoleOutput;
use Throwable;

class Primary extends Command implements CommandInterface, CleanupInterface
{
Expand Down Expand Up @@ -58,32 +58,25 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm
$this->output->writeln("<comment>Exporting database from local MySql</comment>");

try {
$command = $this->commandAssembler->execute($this->localConnection, $environment, $this->getDumpFile());
if (empty($command)) {
$commands = $this->commandAssembler->execute($this->localConnection, $environment, $this->getDumpFile());
if (empty($commands)) {
throw new RuntimeException('Nothing to import');
}

$transport->getLogger()->debug(
"Local connection string: " . str_replace(
$this->localConnection->getPassword(),
'',
$command
)
);
$this->output->writeln("<comment>Local connection string: </comment>" . str_replace(
$this->localConnection->getPassword(),
'',
$command
));

$results = system($command);

if ($results) {
throw new RuntimeException($results);
foreach ($commands as $command) {
$strippedCommand = str_replace($this->localConnection->getPassword(), '', $command);
$transport->getLogger()->debug('Command: ' . $strippedCommand);
$resultCode = 0;
$result = system($command, $resultCode);
if ($result === false || $resultCode !== 0) {
$message = sprintf('Error (%s) when executing command: %s', $resultCode, $strippedCommand);
$this->output->writeln("<error>${$message}</error>");
throw new RuntimeException($message);
}
}
} catch (Exception $e) {
} catch (Throwable $e) {
$this->output->writeln('<error>Import to RDS instance failed: ' . $e->getMessage() . '</error>');
throw new Exception('Import to RDS instance failed: ' . $e->getMessage());
throw new RuntimeException('Import to RDS instance failed: ' . $e->getMessage());
}

$this->logger->notice("Database dump has completed.");
Expand Down
52 changes: 34 additions & 18 deletions src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,62 +37,78 @@ public function setUp(): void
$this->commandAssembler = new CommandAssembler($this->tablesProviderMock);
}

public function testReturnsEmptyStringIfNoTables(): void
public function testReturnsEmptyArrayIfNoTables(): void
{
$this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn([]);
$this->assertSame(
'',
[],
$this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql')
);
}

public function testReturnsEmptyStringIfAllTablesAreIgnored(): void
public function testReturnsEmptyArrayIfAllTablesAreIgnored(): void
{
$this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']);
$this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn(['a', 'b']);
$this->assertSame(
'',
[],
$this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql')
);
}

public function testReturnsCommandForNormalTables(): void
public function testReturnsCommandsForNormalTables(): void
{
$this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']);
$this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn([]);
$this->assertSame(
'mysqldump --user="user" --password="password" --single-transaction --host=host db a >> dump.sql;'
. 'mysqldump --user="user" --password="password" --single-transaction --host=host db b >> dump.sql;'
. "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz",
[
"echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql",
'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host '
. 'db a >> dump.sql',
'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host '
. 'db b >> dump.sql',
"echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql",
"cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz"
],
$this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql')
);
}

public function testReturnsCommandForEmptyTables(): void
public function testReturnsCommandsForEmptyTables(): void
{
$this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']);
$this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn([]);
$this->tablesProviderMock->expects($this->any())->method('getEmptyTables')->willReturn(['a', 'b']);
$this->assertSame(
'mysqldump --user="user" --password="password" --single-transaction --host=host '
. 'db a b --no-data >> dump.sql;'
. "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz",
[
"echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql",
'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host '
. 'db a b --no-data >> dump.sql',
"echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql",
"cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz"
],
$this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql')
);
}

public function testReturnsCommandForMixedTables(): void
public function testReturnsCommandsForMixedTables(): void
{
$this->tablesProviderMock->expects($this->any())->method('getAllTables')
->willReturn(['a', 'b', 'c', 'd', 'e', 'f']);
$this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn(['c', 'f']);
$this->tablesProviderMock->expects($this->any())->method('getEmptyTables')->willReturn(['b', 'e']);
$this->assertSame(
'mysqldump --user="user" --password="password" --single-transaction --host=host db a >> dump.sql;'
. 'mysqldump --user="user" --password="password" --single-transaction --host=host db d >> dump.sql;'
. 'mysqldump --user="user" --password="password" --single-transaction --host=host '
. 'db b e --no-data >> dump.sql;'
. "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz",
[
"echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql",
'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host '
. 'db a >> dump.sql',
'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host '
. 'db d >> dump.sql',
'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host '
. 'db b e --no-data >> dump.sql',
"echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql",
"cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz"
],
$this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql')
);
}
Expand Down

0 comments on commit ae343c6

Please sign in to comment.