Skip to content

Commit

Permalink
support backing up mutli-user data.
Browse files Browse the repository at this point in the history
  • Loading branch information
@arabcoders
arabcoders committed Jan 31, 2025
1 parent 3e23a3b commit d10a03c
Show file tree
Hide file tree
Showing 2 changed files with 162 additions and 49 deletions.
2 changes: 1 addition & 1 deletion src/Backends/Jellyfin/Action/Import.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -947,7 +947,7 @@ protected function handle(Context $context, iResponse $response, Closure $callba

$end = microtime(true);
$this->logger->info(
"Parsing '{client}: {backend}' library '{library.title} {segment.number}/{segment.of}' completed in '{time.duration}'.",
"Parsing '{client}: {backend}' library '{library.title} {segment.number}/{segment.of}' completed in '{time.duration}'s.",
[
'client' => $context->clientName,
'backend' => $context->backendName,
Expand Down
209 changes: 161 additions & 48 deletions src/Commands/State/BackupCommand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,22 @@

namespace App\Commands\State;

use App\Backends\Common\Cache as BackendCache;
use App\Command;
use App\Libs\Attributes\Route\Cli;
use App\Libs\Config;
use App\Libs\ConfigFile;
use App\Libs\Container;
use App\Libs\Mappers\ExtendedImportInterface as iEImport;
use App\Libs\Mappers\Import\DirectMapper;
use App\Libs\Options;
use App\Libs\Stream;
use Psr\Http\Message\StreamInterface;
use Psr\Log\LoggerInterface;
use Symfony\Component\Console\Input\InputInterface;
use DirectoryIterator;
use Psr\Http\Message\StreamInterface as iStream;
use Psr\Log\LoggerInterface as iLogger;
use Symfony\Component\Console\Input\InputInterface as iInput;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Output\OutputInterface as iOutput;
use Symfony\Contracts\HttpClient\ResponseInterface;
use Throwable;

Expand All @@ -34,9 +39,9 @@ class BackupCommand extends Command
* Constructs a new instance of the class.
*
* @param DirectMapper $mapper The direct mapper instance.
* @param LoggerInterface $logger The logger instance.
* @param iLogger $logger The logger instance.
*/
public function __construct(private DirectMapper $mapper, private LoggerInterface $logger)
public function __construct(private DirectMapper $mapper, private iLogger $logger)
{
set_time_limit(0);
ini_set('memory_limit', '-1');
Expand Down Expand Up @@ -78,6 +83,7 @@ protected function configure(): void
InputOption::VALUE_REQUIRED,
'Full path backup file. Will only be used if backup list is 1'
)
->addOption('only-main-user', 'M', InputOption::VALUE_NONE, 'Only backup main user data.')
->addOption('no-compress', 'N', InputOption::VALUE_NONE, 'Do not compress the backup file.')
->setHelp(
r(
Expand Down Expand Up @@ -138,31 +144,92 @@ protected function configure(): void
/**
* Make sure the command is not running in parallel.
*
* @param InputInterface $input The input interface instance.
* @param OutputInterface $output The output interface instance.
* @param iInput $input The input interface instance.
* @param iOutput $output The output interface instance.
*
* @return int The exit code of the command.
*/
protected function runCommand(InputInterface $input, OutputInterface $output): int
protected function runCommand(iInput $input, iOutput $output): int
{
return $this->single(fn(): int => $this->process($input), $output);
}

private function getBackends(iInput $input): array
{
$configs = [
'main' => [
'config' => ConfigFile::open(Config::get('backends_file'), 'yaml'),
'mapper' => $this->mapper,
'cache' => null,
]
];

if (true === $input->getOption('only-main-user')) {
return $configs;
}

$usersDir = Config::get('path') . '/users';

if (false === is_dir($usersDir)) {
return $configs;
}

if (!is_readable($usersDir)) {
$this->logger->error("SYSTEM: Unable to read '{dir}' directory.", ['dir' => $usersDir]);
return $configs;
}

$mainUserIds = array_map(fn($backend) => ag($backend, 'user'), ag($configs, 'main.config')->getAll());

foreach (new DirectoryIterator(Config::get('path') . '/users') as $dir) {
if ($dir->isDot() || false === $dir->isDir()) {
continue;
}

$config = perUserConfig($dir->getBasename());
$subUserIds = array_map(fn($backend) => ag($backend, 'user'), $config->getAll());

foreach ($mainUserIds as $mainId) {
if (false === in_array($mainId, $subUserIds)) {
continue;
}

$this->logger->debug("SYSTEM: Skipping '{user}' backends as it's same as main user.", [
'user' => $dir->getBasename(),
'main' => $mainUserIds,
'sub' => $subUserIds,
]);
continue 2;
}

$userName = $dir->getBasename();
$perUserCache = perUserCacheAdapter($userName);

$configs[$userName] = [
'config' => $config,
'mapper' => $this->mapper->withDB(perUserDb($userName))
->withCache($perUserCache)
->withLogger($this->logger)
->withOptions(
array_replace_recursive($this->mapper->getOptions(), [Options::ALT_NAME => $userName])
)
->loadData(),
'cache' => $perUserCache,
];
}

return $configs;
}

/**
* Execute the command.
*
* @param InputInterface $input The input interface.
* @param iInput $input The input interface.
*
* @return int The integer result.
*/
protected function process(InputInterface $input): int
protected function process(iInput $input): int
{
$list = [];
$selected = $input->getOption('select-backend');
$isCustom = !empty($selected) && count($selected) > 0;
$supported = Config::get('supported', []);
$noCompression = $input->getOption('no-compress');

$mapperOpts = [];

if ($input->getOption('dry-run')) {
Expand All @@ -178,27 +245,55 @@ protected function process(InputInterface $input): int
$this->mapper->setOptions(options: $mapperOpts);
}

foreach (Config::get('servers', []) as $backendName => $backend) {
$this->logger->notice("Using WatchState version - '{version}'.", ['version' => getAppVersion()]);
foreach ($this->getBackends($input) as $user => $opt) {
try {
$this->process_backup($input, $user, $opt);
} finally {
ag($opt, 'mapper')->reset();
}
}

return self::SUCCESS;
}

private function process_backup(iInput $input, string $user, array $opt): void
{
$list = [];

$selected = $input->getOption('select-backend');
$isCustom = !empty($selected) && count($selected) > 0;
$supported = Config::get('supported', []);

$noCompression = $input->getOption('no-compress');

$config = ag($opt, 'config');
assert($config instanceof ConfigFile);

foreach ($config->getAll() as $backendName => $backend) {
$type = strtolower(ag($backend, 'type', 'unknown'));

if ($isCustom && $input->getOption('exclude') === in_array($backendName, $selected, true)) {
$this->logger->info("SYSTEM: Ignoring '{backend}' as requested by [-s, --select-backend].", [
if ($isCustom && $input->getOption('exclude') === $this->in_array($selected, $backendName)) {
$this->logger->info("SYSTEM: Ignoring '{user}@{backend}' as requested by [-s, --select-backend].", [
'user' => $user,
'backend' => $backendName
]);
continue;
}

if (true !== (bool)ag($backend, 'import.enabled')) {
$this->logger->info("SYSTEM: Ignoring '{backend}' as the backend has import disabled.", [
$this->logger->info("SYSTEM: Ignoring '{user}@{backend}' as the backend has import disabled.", [
'user' => $user,
'backend' => $backendName
]);
continue;
}

if (!isset($supported[$type])) {
$this->logger->error(
"SYSTEM: Ignoring '{backend}' due to unexpected type '{type}'. Expecting '{types}'.",
"SYSTEM: Ignoring '{user}@{backend}' due to unexpected type '{type}'. Expecting '{types}'.",
[
'user' => $user,
'type' => $type,
'backend' => $backendName,
'types' => implode(', ', array_keys($supported)),
Expand All @@ -208,7 +303,8 @@ protected function process(InputInterface $input): int
}

if (null === ($url = ag($backend, 'url')) || false === isValidURL($url)) {
$this->logger->error("SYSTEM: Ignoring '{backend}' due to invalid URL. '{url}'.", [
$this->logger->error("SYSTEM: Ignoring '{user}@{backend}' due to invalid URL. '{url}'.", [
'user' => $user,
'url' => $url ?? 'None',
'backend' => $backendName,
]);
Expand All @@ -223,12 +319,16 @@ protected function process(InputInterface $input): int
$this->logger->warning(
$isCustom ? '[-s, --select-backend] flag did not match any backend.' : 'No backends were found.'
);
return self::FAILURE;
return;
}

$mapper = ag($opt, 'mapper');
assert($mapper instanceof iEImport);

if (true !== $input->getOption('no-enhance')) {
$this->logger->notice("SYSTEM: Preloading '{mapper}' data.", [
'mapper' => afterLast($this->mapper::class, '\\'),
$this->logger->notice("SYSTEM: Preloading '{user}@{mapper}' data.", [
'user' => $user,
'mapper' => afterLast($mapper::class, '\\'),
'memory' => [
'now' => getMemoryUsage(),
'peak' => getPeakMemoryUsage(),
Expand All @@ -238,7 +338,8 @@ protected function process(InputInterface $input): int
$start = microtime(true);
$this->mapper->loadData();

$this->logger->notice("SYSTEM: Preloading '{mapper}' data completed in '{duration}s'.", [
$this->logger->notice("SYSTEM: Preloading '{user}@{mapper}' data completed in '{duration}s'.", [
'user' => $user,
'mapper' => afterLast($this->mapper::class, '\\'),
'duration' => round(microtime(true) - $start, 2),
'memory' => [
Expand All @@ -251,8 +352,6 @@ protected function process(InputInterface $input): int
/** @var array<array-key,ResponseInterface> $queue */
$queue = [];

$this->logger->notice("Using WatchState version - '{version}'.", ['version' => getAppVersion()]);

foreach ($list as $name => &$backend) {
$opts = ag($backend, 'options', []);

Expand All @@ -269,9 +368,19 @@ protected function process(InputInterface $input): int
}

$backend['options'] = $opts;
$backend['class'] = $this->getBackend($name, $backend);

$this->logger->notice('SYSTEM: Backing up [{backend}] play state.', [
$backendOpts = [];

if (null !== ag($opt, 'cache')) {
$backendOpts = [
BackendCache::class => Container::get(BackendCache::class)->with(adapter: ag($opt, 'cache')),
];
}

$backend['class'] = makeBackend($backend, $name, $backendOpts)->setLogger($this->logger);

$this->logger->notice("SYSTEM: Backing up '{user}@{backend}' play state.", [
'user' => $user,
'backend' => $name,
]);

Expand All @@ -297,7 +406,8 @@ protected function process(InputInterface $input): int
touch($fileName);
}

$this->logger->notice("SYSTEM: '{backend}' is using '{file}' as backup target.", [
$this->logger->notice("SYSTEM: '{user}@{backend}' is using '{file}' as backup target.", [
'user' => $user,
'file' => realpath($fileName),
'backend' => $name,
]);
Expand All @@ -306,24 +416,19 @@ protected function process(InputInterface $input): int
$backend['fp']->write('[');
}

array_push(
$queue,
...$backend['class']->backup(
$this->mapper,
$backend['fp'] ?? null,
[
'no_enhance' => true === $input->getOption('no-enhance'),
Options::DRY_RUN => (bool)$input->getOption('dry-run'),
]
)
);
array_push($queue, ...$backend['class']->backup($mapper, $backend['fp'] ?? null, [
'no_enhance' => true === $input->getOption('no-enhance'),
Options::DRY_RUN => (bool)$input->getOption('dry-run'),
]));
}

unset($backend);

$start = microtime(true);
$this->logger->notice("SYSTEM: Waiting on '{total}' requests.", [
$this->logger->notice("SYSTEM: Waiting on '{total}' requests for '{user}: {backends}' backends.", [
'user' => $user,
'total' => number_format(count($queue)),
'backends' => implode(', ', array_keys($list)),
'memory' => [
'now' => getMemoryUsage(),
'peak' => getPeakMemoryUsage(),
Expand All @@ -349,15 +454,18 @@ protected function process(InputInterface $input): int
continue;
}

assert($backend['fp'] instanceof StreamInterface);
assert($backend['fp'] instanceof iStream);

if (false === $input->getOption('dry-run')) {
$backend['fp']->seek(-1, SEEK_END);
$backend['fp']->write(PHP_EOL . ']');

if (false === $noCompression) {
$file = $backend['fp']->getMetadata('uri');
$this->logger->notice("SYSTEM: Compressing '{file}'.", ['file' => $file]);
$this->logger->notice("SYSTEM: Compressing '{user}@{file}'.", [
'user' => $user,
'file' => $file
]);
$status = compress_files($file, [$file], ['affix' => 'zip']);
if (true === $status) {
unlink($file);
Expand All @@ -368,14 +476,19 @@ protected function process(InputInterface $input): int
}
}

$this->logger->notice("SYSTEM: Backup operation finished in '{duration}s'.", [
$this->logger->notice("SYSTEM: Backup operation for '{user}: {backends}' backends finished in '{duration}s'.", [
'user' => $user,
'backends' => implode(', ', array_keys($list)),
'duration' => round(microtime(true) - $start, 2),
'memory' => [
'now' => getMemoryUsage(),
'peak' => getPeakMemoryUsage(),
],
]);
}

return self::SUCCESS;
private function in_array(array $haystack, string $needle): bool
{
return array_any($haystack, fn($item) => str_starts_with($item, $needle));
}
}

0 comments on commit d10a03c

Please sign in to comment.