diff --git a/app/Actions/Photo/Archive.php b/app/Actions/Photo/Archive.php index 6ab95932d64..326da21980c 100644 --- a/app/Actions/Photo/Archive.php +++ b/app/Actions/Photo/Archive.php @@ -4,7 +4,6 @@ use App\Actions\Photo\Extensions\ArchiveFileInfo; use App\Contracts\Exceptions\LycheeException; -use App\Contracts\Models\AbstractSizeVariantNamingStrategy; use App\Enum\DownloadVariantType; use App\Enum\SizeVariantType; use App\Exceptions\ConfigurationKeyMissingException; @@ -14,6 +13,7 @@ use App\Models\Configs; use App\Models\Photo; use Illuminate\Database\Eloquent\Collection; +use Illuminate\Support\Facades\Storage; use Safe\Exceptions\InfoException; use function Safe\fclose; use function Safe\fopen; @@ -284,7 +284,8 @@ protected function extractFileInfo(Photo $photo, DownloadVariantType $downloadVa $baseFilename = $validFilename !== '' ? $validFilename : 'Untitled'; if ($downloadVariant === DownloadVariantType::LIVEPHOTOVIDEO) { - $sourceFile = new FlysystemFile(AbstractSizeVariantNamingStrategy::getImageDisk(), $photo->live_photo_short_path); + $disk = $photo->size_variants->getSizeVariant(SizeVariantType::ORIGINAL)->storage_disk->value; + $sourceFile = new FlysystemFile(Storage::disk($disk), $photo->live_photo_short_path); $baseFilenameAddon = ''; } else { $sv = $photo->size_variants->getSizeVariant($downloadVariant->getSizeVariantType()); diff --git a/app/Actions/Photo/Create.php b/app/Actions/Photo/Create.php index 5f44a965b4d..5fa64921b05 100644 --- a/app/Actions/Photo/Create.php +++ b/app/Actions/Photo/Create.php @@ -160,6 +160,7 @@ private function handleStandalone(InitDTO $initDTO): Photo Shared\Save::class, Standalone\CreateOriginalSizeVariant::class, Standalone\CreateSizeVariants::class, + Shared\UploadSizeVariantsToS3::class, ]; return $this->executePipeOnDTO($pipes, $dto)->getPhoto(); @@ -244,6 +245,7 @@ private function handlePhotoLivePartner(InitDTO $initDTO): Photo Shared\Save::class, Standalone\CreateOriginalSizeVariant::class, Standalone\CreateSizeVariants::class, + Shared\UploadSizeVariantsToS3::class, ]; $standAloneDto = $this->executePipeOnDTO($standAlonePipes, $standAloneDto); diff --git a/app/Actions/Photo/Pipes/Shared/UploadSizeVariantsToS3.php b/app/Actions/Photo/Pipes/Shared/UploadSizeVariantsToS3.php new file mode 100644 index 00000000000..3f4c350a300 --- /dev/null +++ b/app/Actions/Photo/Pipes/Shared/UploadSizeVariantsToS3.php @@ -0,0 +1,33 @@ +getPhoto()->size_variants->toCollection() + ->filter(fn ($v) => $v !== null) + ->map(fn (SizeVariant $variant) => new UploadSizeVariantToS3Job($variant)); + + $jobs->each(fn ($job) => $use_job_queues ? dispatch($job) : dispatch_sync($job)); + } + + return $next($state); + } +} diff --git a/app/Actions/Photo/Pipes/VideoPartner/PlaceVideo.php b/app/Actions/Photo/Pipes/VideoPartner/PlaceVideo.php index 8b294a2e4cb..a40fe4175cd 100644 --- a/app/Actions/Photo/Pipes/VideoPartner/PlaceVideo.php +++ b/app/Actions/Photo/Pipes/VideoPartner/PlaceVideo.php @@ -3,9 +3,10 @@ namespace App\Actions\Photo\Pipes\VideoPartner; use App\Actions\Diagnostics\Pipes\Checks\BasicPermissionCheck; -use App\Contracts\Models\AbstractSizeVariantNamingStrategy; +use App\Assets\Features; use App\Contracts\PhotoCreate\VideoPartnerPipe; use App\DTO\PhotoCreate\VideoPartnerDTO; +use App\Enum\StorageDiskType; use App\Exceptions\ConfigurationException; use App\Exceptions\Handler; use App\Exceptions\Internal\LycheeAssertionError; @@ -13,6 +14,7 @@ use App\Image\Files\FlysystemFile; use App\Image\Files\NativeLocalFile; use App\Image\StreamStat; +use Illuminate\Support\Facades\Storage; /** * Puts the video source file into the final position at video target file. @@ -47,7 +49,11 @@ class PlaceVideo implements VideoPartnerPipe { public function handle(VideoPartnerDTO $state, \Closure $next): VideoPartnerDTO { - $videoTargetFile = new FlysystemFile(AbstractSizeVariantNamingStrategy::getImageDisk(), $state->videoPath); + $disk = Storage::disk(StorageDiskType::LOCAL->value); + if (Features::active('use-s3')) { + $disk = Storage::disk(StorageDiskType::S3->value); + } + $videoTargetFile = new FlysystemFile($disk, $state->videoPath); try { if ($state->videoFile instanceof NativeLocalFile) { diff --git a/app/Console/Commands/Ghostbuster.php b/app/Console/Commands/Ghostbuster.php index a9d04244d8e..26479f5033e 100644 --- a/app/Console/Commands/Ghostbuster.php +++ b/app/Console/Commands/Ghostbuster.php @@ -2,9 +2,10 @@ namespace App\Console\Commands; +use App\Assets\Features; use App\Console\Commands\Utilities\Colorize; -use App\Contracts\Models\AbstractSizeVariantNamingStrategy; use App\Enum\SizeVariantType; +use App\Enum\StorageDiskType; use App\Exceptions\UnexpectedException; use App\Models\Photo; use App\Models\SizeVariant; @@ -73,11 +74,17 @@ public function handle(): int $removeDeadSymLinks = filter_var($this->option('removeDeadSymLinks'), FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE) === true; $removeZombiePhotos = filter_var($this->option('removeZombiePhotos'), FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE) === true; $dryrun = filter_var($this->option('dryrun'), FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE) !== false; - $uploadDisk = AbstractSizeVariantNamingStrategy::getImageDisk(); + $uploadDisk = Features::active('use-s3') + ? Storage::disk(StorageDiskType::S3->value) + : Storage::disk(StorageDiskType::LOCAL->value); $symlinkDisk = Storage::disk(SymLink::DISK_NAME); $isLocalDisk = $uploadDisk->getAdapter() instanceof LocalFilesystemAdapter; $this->line(''); + if (!$isLocalDisk) { + $this->line($this->col->red('Using non-local disk to store images, USE AT YOUR OWN RISKS! This code is not battle tested.')); + $this->line(''); + } if ($removeDeadSymLinks && !$isLocalDisk) { $this->line($this->col->yellow('Removal of dead symlinks requested, but filesystem does not support symlinks.')); diff --git a/app/Contracts/Models/AbstractSizeVariantNamingStrategy.php b/app/Contracts/Models/AbstractSizeVariantNamingStrategy.php index c5f02cde656..38d88983e9b 100644 --- a/app/Contracts/Models/AbstractSizeVariantNamingStrategy.php +++ b/app/Contracts/Models/AbstractSizeVariantNamingStrategy.php @@ -6,32 +6,15 @@ use App\Enum\SizeVariantType; use App\Image\Files\FlysystemFile; use App\Models\Photo; -use Illuminate\Filesystem\FilesystemAdapter; -use Illuminate\Support\Facades\Storage; /** * Interface SizeVariantNamingStrategy. */ abstract class AbstractSizeVariantNamingStrategy { - /** - * The name of the Flysystem disk where images are stored. - */ - public const IMAGE_DISK_NAME = 'images'; - protected string $extension = ''; protected ?Photo $photo = null; - /** - * Returns the disk on which the size variants are put. - * - * @return FilesystemAdapter - */ - public static function getImageDisk(): FilesystemAdapter - { - return Storage::disk(self::IMAGE_DISK_NAME); - } - /** * Sets the extension to be used for the size variants. * diff --git a/app/Http/Kernel.php b/app/Http/Kernel.php index 4f5c55c327d..74b55da01b7 100644 --- a/app/Http/Kernel.php +++ b/app/Http/Kernel.php @@ -89,7 +89,6 @@ class Kernel extends HttpKernel 'installation' => \App\Http\Middleware\InstallationStatus::class, 'admin_user' => \App\Http\Middleware\AdminUserStatus::class, 'migration' => \App\Http\Middleware\MigrationStatus::class, - 'local_storage' => \App\Http\Middleware\LocalStorageOnly::class, 'content_type' => \App\Http\Middleware\ContentType::class, 'accept_content_type' => \App\Http\Middleware\AcceptContentType::class, 'redirect-legacy-id' => \App\Http\Middleware\RedirectLegacyPhotoID::class, diff --git a/app/Http/Middleware/LocalStorageOnly.php b/app/Http/Middleware/LocalStorageOnly.php deleted file mode 100644 index 5f336c70b29..00000000000 --- a/app/Http/Middleware/LocalStorageOnly.php +++ /dev/null @@ -1,31 +0,0 @@ -getAdapter(); - if (!($storageAdapter instanceof LocalFilesystemAdapter)) { - throw new RequestUnsupportedException($request->url() . ' not implemented for non-local storage'); - } - - return $next($request); - } -} diff --git a/app/Jobs/UploadSizeVariantToS3Job.php b/app/Jobs/UploadSizeVariantToS3Job.php new file mode 100644 index 00000000000..455991371c2 --- /dev/null +++ b/app/Jobs/UploadSizeVariantToS3Job.php @@ -0,0 +1,96 @@ +variant = $variant; + + // Set up our new history record. + $this->history = new JobHistory(); + $this->history->owner_id = Auth::user()->id; + $this->history->job = Str::limit(sprintf('Upload sizeVariant to S3: %s.', $this->variant->short_path), 200); + $this->history->status = JobStatus::READY; + $this->history->save(); + } + + public function handle(): void + { + $this->history->status = JobStatus::STARTED; + $this->history->save(); + + Storage::disk(StorageDiskType::S3->value)->writeStream( + $this->variant->short_path, + Storage::disk(StorageDiskType::LOCAL->value)->readStream($this->variant->short_path) + ); + + Storage::disk(StorageDiskType::LOCAL->value)->delete($this->variant->short_path); + + $this->variant->storage_disk = StorageDiskType::S3; + $this->variant->save(); + + $this->handleVideoPartner(); + + // Once the job has finished, set history status to 1. + $this->history->status = JobStatus::SUCCESS; + $this->history->save(); + } + + public function failed(\Throwable $th): void + { + $this->history->status = JobStatus::FAILURE; + $this->history->save(); + + if ($th->getCode() === 999) { + $this->release(); + } else { + Log::error(__LINE__ . ':' . __FILE__ . ' ' . $th->getMessage(), $th->getTrace()); + } + } + + /** + * If we have a live partner, then we also upload it. + */ + private function handleVideoPartner(): void + { + if ($this->variant->type !== SizeVariantType::ORIGINAL) { + return; + } + + $photo = Photo::query()->where('id', '=', $this->variant->photo_id)->first(); + + Storage::disk(StorageDiskType::S3->value)->writeStream( + $photo->live_photo_short_path, + Storage::disk(StorageDiskType::LOCAL->value)->readStream($photo->live_photo_short_path) + ); + + Storage::disk(StorageDiskType::LOCAL->value)->delete($photo->live_photo_short_path); + } +} diff --git a/app/Legacy/Actions/Photo/Strategies/AddStandaloneStrategy.php b/app/Legacy/Actions/Photo/Strategies/AddStandaloneStrategy.php index bc05fc73038..fcdfb2f862f 100644 --- a/app/Legacy/Actions/Photo/Strategies/AddStandaloneStrategy.php +++ b/app/Legacy/Actions/Photo/Strategies/AddStandaloneStrategy.php @@ -3,6 +3,7 @@ namespace App\Legacy\Actions\Photo\Strategies; use App\Actions\Diagnostics\Pipes\Checks\BasicPermissionCheck; +use App\Assets\Features; use App\Contracts\Exceptions\LycheeException; use App\Contracts\Image\ImageHandlerInterface; use App\Contracts\Image\StreamStats; @@ -22,8 +23,10 @@ use App\Image\Handlers\ImageHandler; use App\Image\Handlers\VideoHandler; use App\Image\StreamStat; +use App\Jobs\UploadSizeVariantToS3Job; use App\Models\Configs; use App\Models\Photo; +use App\Models\SizeVariant; use Illuminate\Contracts\Container\BindingResolutionException; class AddStandaloneStrategy extends AbstractAddStrategy @@ -143,6 +146,7 @@ public function do(): Photo // we must move the preliminary extracted video file next to the // final target file if ($tmpVideoFile !== null) { + // @TODO S3 How should live videos be handled? $videoTargetPath = pathinfo($targetFile->getRelativePath(), PATHINFO_DIRNAME) . '/' . @@ -169,7 +173,7 @@ public function do(): Photo $imageDim = $this->sourceImage?->isLoaded() ? $this->sourceImage->getDimensions() : new ImageDimension($this->parameters->exifInfo->width, $this->parameters->exifInfo->height); - $this->photo->size_variants->create( + $originalVariant = $this->photo->size_variants->create( SizeVariantType::ORIGINAL, $targetFile->getRelativePath(), $imageDim, @@ -193,7 +197,20 @@ public function do(): Photo /** @var SizeVariantFactory $sizeVariantFactory */ $sizeVariantFactory = resolve(SizeVariantFactory::class); $sizeVariantFactory->init($this->photo, $this->sourceImage, $this->namingStrategy); - $sizeVariantFactory->createSizeVariants(); + $variants = $sizeVariantFactory->createSizeVariants(); + $variants->push($originalVariant); + + if (Features::active('use-s3')) { + // If enabled, upload all size variants to the remote bucket and delete the local files after that + $variants->each(function (SizeVariant $variant) { + if (Configs::getValueAsBool('use_job_queues')) { + UploadSizeVariantToS3Job::dispatch($variant); + } else { + $job = new UploadSizeVariantToS3Job($variant); + $job->handle(); + } + }); + } } catch (\Throwable $t) { // Don't re-throw the exception, because we do not want the // import to fail completely only due to missing size variants. diff --git a/routes/api.php b/routes/api.php index 25898ad9c33..f56d05497b2 100644 --- a/routes/api.php +++ b/routes/api.php @@ -45,7 +45,7 @@ Route::get('/Album::getArchive', [AlbumController::class, 'getArchive']) ->name('download') ->withoutMiddleware(['content_type:json', 'accept_content_type:json']) - ->middleware(['local_storage', 'accept_content_type:any']); + ->middleware(['accept_content_type:any']); Route::post('/Album::setTrack', [AlbumController::class, 'setTrack']) ->withoutMiddleware(['content_type:json']) ->middleware(['content_type:multipart']); @@ -86,7 +86,7 @@ Route::get('/Photo::getArchive', [PhotoController::class, 'getArchive']) ->name('photo_download') ->withoutMiddleware(['content_type:json', 'accept_content_type:json']) - ->middleware(['local_storage', 'accept_content_type:any']); + ->middleware(['accept_content_type:any']); /** * SEARCH.