[Stacked 5] Remove local requirement Middleware (#2383)

* [Stacked 6] Remove hard coded values from naming strategy (#2384)
* [Stacked 7] Finally upload the data to S3 (#2385)

app/Actions/Photo/Archive.php

@@ -4,7 +4,6 @@
 
 use App\Actions\Photo\Extensions\ArchiveFileInfo;
 use App\Contracts\Exceptions\LycheeException;
-use App\Contracts\Models\AbstractSizeVariantNamingStrategy;
 use App\Enum\DownloadVariantType;
 use App\Enum\SizeVariantType;
 use App\Exceptions\ConfigurationKeyMissingException;
@@ -14,6 +13,7 @@
 use App\Models\Configs;
 use App\Models\Photo;
 use Illuminate\Database\Eloquent\Collection;
+use Illuminate\Support\Facades\Storage;
 use Safe\Exceptions\InfoException;
 use function Safe\fclose;
 use function Safe\fopen;
@@ -284,7 +284,8 @@ protected function extractFileInfo(Photo $photo, DownloadVariantType $downloadVa
 		$baseFilename = $validFilename !== '' ? $validFilename : 'Untitled';
 
 		if ($downloadVariant === DownloadVariantType::LIVEPHOTOVIDEO) {
-			$sourceFile = new FlysystemFile(AbstractSizeVariantNamingStrategy::getImageDisk(), $photo->live_photo_short_path);
+			$disk = $photo->size_variants->getSizeVariant(SizeVariantType::ORIGINAL)->storage_disk->value;
+			$sourceFile = new FlysystemFile(Storage::disk($disk), $photo->live_photo_short_path);
 			$baseFilenameAddon = '';
 		} else {
 			$sv = $photo->size_variants->getSizeVariant($downloadVariant->getSizeVariantType());

app/Actions/Photo/Create.php

@@ -160,6 +160,7 @@ private function handleStandalone(InitDTO $initDTO): Photo
 			Shared\Save::class,
 			Standalone\CreateOriginalSizeVariant::class,
 			Standalone\CreateSizeVariants::class,
+			Shared\UploadSizeVariantsToS3::class,
 		];
 
 		return $this->executePipeOnDTO($pipes, $dto)->getPhoto();
@@ -244,6 +245,7 @@ private function handlePhotoLivePartner(InitDTO $initDTO): Photo
 			Shared\Save::class,
 			Standalone\CreateOriginalSizeVariant::class,
 			Standalone\CreateSizeVariants::class,
+			Shared\UploadSizeVariantsToS3::class,
 		];
 		$standAloneDto = $this->executePipeOnDTO($standAlonePipes, $standAloneDto);
 

app/Actions/Photo/Pipes/Shared/UploadSizeVariantsToS3.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace App\Actions\Photo\Pipes\Shared;
+
+use App\Assets\Features;
+use App\Contracts\PhotoCreate\PhotoDTO;
+use App\Contracts\PhotoCreate\PhotoPipe;
+use App\Jobs\UploadSizeVariantToS3Job;
+use App\Models\Configs;
+use App\Models\SizeVariant;
+
+/**
+ * Upload SizeVariants to S3 once done (if enabled).
+ * Note that we first create the job, then we dispatch it.
+ * This allows to manage the queue properly and see it in the feedback.
+ */
+class UploadSizeVariantsToS3 implements PhotoPipe
+{
+	public function handle(PhotoDTO $state, \Closure $next): PhotoDTO
+	{
+		if (Features::active('use-s3')) {
+			$use_job_queues = Configs::getValueAsBool('use_job_queues');
+
+			$jobs = $state->getPhoto()->size_variants->toCollection()
+				->filter(fn ($v) => $v !== null)
+				->map(fn (SizeVariant $variant) => new UploadSizeVariantToS3Job($variant));
+
+			$jobs->each(fn ($job) => $use_job_queues ? dispatch($job) : dispatch_sync($job));
+		}
+
+		return $next($state);
+	}
+}

app/Actions/Photo/Pipes/VideoPartner/PlaceVideo.php

@@ -3,16 +3,18 @@
 namespace App\Actions\Photo\Pipes\VideoPartner;
 
 use App\Actions\Diagnostics\Pipes\Checks\BasicPermissionCheck;
-use App\Contracts\Models\AbstractSizeVariantNamingStrategy;
+use App\Assets\Features;
 use App\Contracts\PhotoCreate\VideoPartnerPipe;
 use App\DTO\PhotoCreate\VideoPartnerDTO;
+use App\Enum\StorageDiskType;
 use App\Exceptions\ConfigurationException;
 use App\Exceptions\Handler;
 use App\Exceptions\Internal\LycheeAssertionError;
 use App\Exceptions\MediaFileOperationException;
 use App\Image\Files\FlysystemFile;
 use App\Image\Files\NativeLocalFile;
 use App\Image\StreamStat;
+use Illuminate\Support\Facades\Storage;
 
 /**
  * Puts the video source file into the final position at video target file.
@@ -47,7 +49,11 @@ class PlaceVideo implements VideoPartnerPipe
 {
 	public function handle(VideoPartnerDTO $state, \Closure $next): VideoPartnerDTO
 	{
-		$videoTargetFile = new FlysystemFile(AbstractSizeVariantNamingStrategy::getImageDisk(), $state->videoPath);
+		$disk = Storage::disk(StorageDiskType::LOCAL->value);
+		if (Features::active('use-s3')) {
+			$disk = Storage::disk(StorageDiskType::S3->value);
+		}
+		$videoTargetFile = new FlysystemFile($disk, $state->videoPath);
 
 		try {
 			if ($state->videoFile instanceof NativeLocalFile) {

app/Console/Commands/Ghostbuster.php

@@ -2,9 +2,10 @@
 
 namespace App\Console\Commands;
 
+use App\Assets\Features;
 use App\Console\Commands\Utilities\Colorize;
-use App\Contracts\Models\AbstractSizeVariantNamingStrategy;
 use App\Enum\SizeVariantType;
+use App\Enum\StorageDiskType;
 use App\Exceptions\UnexpectedException;
 use App\Models\Photo;
 use App\Models\SizeVariant;
@@ -73,11 +74,17 @@ public function handle(): int
 			$removeDeadSymLinks = filter_var($this->option('removeDeadSymLinks'), FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE) === true;
 			$removeZombiePhotos = filter_var($this->option('removeZombiePhotos'), FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE) === true;
 			$dryrun = filter_var($this->option('dryrun'), FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE) !== false;
-			$uploadDisk = AbstractSizeVariantNamingStrategy::getImageDisk();
+			$uploadDisk = Features::active('use-s3')
+				? Storage::disk(StorageDiskType::S3->value)
+				: Storage::disk(StorageDiskType::LOCAL->value);
 			$symlinkDisk = Storage::disk(SymLink::DISK_NAME);
 			$isLocalDisk = $uploadDisk->getAdapter() instanceof LocalFilesystemAdapter;
 
 			$this->line('');
+			if (!$isLocalDisk) {
+				$this->line($this->col->red('Using non-local disk to store images, USE AT YOUR OWN RISKS! This code is not battle tested.'));
+				$this->line('');
+			}
 
 			if ($removeDeadSymLinks && !$isLocalDisk) {
 				$this->line($this->col->yellow('Removal of dead symlinks requested, but filesystem does not support symlinks.'));

app/Contracts/Models/AbstractSizeVariantNamingStrategy.php

@@ -6,32 +6,15 @@
 use App\Enum\SizeVariantType;
 use App\Image\Files\FlysystemFile;
 use App\Models\Photo;
-use Illuminate\Filesystem\FilesystemAdapter;
-use Illuminate\Support\Facades\Storage;
 
 /**
  * Interface SizeVariantNamingStrategy.
  */
 abstract class AbstractSizeVariantNamingStrategy
 {
-	/**
-	 * The name of the Flysystem disk where images are stored.
-	 */
-	public const IMAGE_DISK_NAME = 'images';
-
 	protected string $extension = '';
 	protected ?Photo $photo = null;
 
-	/**
-	 * Returns the disk on which the size variants are put.
-	 *
-	 * @return FilesystemAdapter
-	 */
-	public static function getImageDisk(): FilesystemAdapter
-	{
-		return Storage::disk(self::IMAGE_DISK_NAME);
-	}
-
 	/**
 	 * Sets the extension to be used for the size variants.
 	 *

app/Http/Kernel.php

@@ -89,7 +89,6 @@ class Kernel extends HttpKernel
 		'installation' => \App\Http\Middleware\InstallationStatus::class,
 		'admin_user' => \App\Http\Middleware\AdminUserStatus::class,
 		'migration' => \App\Http\Middleware\MigrationStatus::class,
-		'local_storage' => \App\Http\Middleware\LocalStorageOnly::class,
 		'content_type' => \App\Http\Middleware\ContentType::class,
 		'accept_content_type' => \App\Http\Middleware\AcceptContentType::class,
 		'redirect-legacy-id' => \App\Http\Middleware\RedirectLegacyPhotoID::class,

app/Jobs/UploadSizeVariantToS3Job.php

@@ -0,0 +1,96 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Enum\JobStatus;
+use App\Enum\SizeVariantType;
+use App\Enum\StorageDiskType;
+use App\Models\JobHistory;
+use App\Models\Photo;
+use App\Models\SizeVariant;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Auth;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Storage;
+use Illuminate\Support\Str;
+
+class UploadSizeVariantToS3Job implements ShouldQueue
+{
+	use Dispatchable;
+	use InteractsWithQueue;
+	use Queueable;
+	use SerializesModels;
+
+	protected JobHistory $history;
+
+	protected SizeVariant $variant;
+
+	public function __construct(SizeVariant $variant)
+	{
+		$this->variant = $variant;
+
+		// Set up our new history record.
+		$this->history = new JobHistory();
+		$this->history->owner_id = Auth::user()->id;
+		$this->history->job = Str::limit(sprintf('Upload sizeVariant to S3: %s.', $this->variant->short_path), 200);
+		$this->history->status = JobStatus::READY;
+		$this->history->save();
+	}
+
+	public function handle(): void
+	{
+		$this->history->status = JobStatus::STARTED;
+		$this->history->save();
+
+		Storage::disk(StorageDiskType::S3->value)->writeStream(
+			$this->variant->short_path,
+			Storage::disk(StorageDiskType::LOCAL->value)->readStream($this->variant->short_path)
+		);
+
+		Storage::disk(StorageDiskType::LOCAL->value)->delete($this->variant->short_path);
+
+		$this->variant->storage_disk = StorageDiskType::S3;
+		$this->variant->save();
+
+		$this->handleVideoPartner();
+
+		// Once the job has finished, set history status to 1.
+		$this->history->status = JobStatus::SUCCESS;
+		$this->history->save();
+	}
+
+	public function failed(\Throwable $th): void
+	{
+		$this->history->status = JobStatus::FAILURE;
+		$this->history->save();
+
+		if ($th->getCode() === 999) {
+			$this->release();
+		} else {
+			Log::error(__LINE__ . ':' . __FILE__ . ' ' . $th->getMessage(), $th->getTrace());
+		}
+	}
+
+	/**
+	 * If we have a live partner, then we also upload it.
+	 */
+	private function handleVideoPartner(): void
+	{
+		if ($this->variant->type !== SizeVariantType::ORIGINAL) {
+			return;
+		}
+
+		$photo = Photo::query()->where('id', '=', $this->variant->photo_id)->first();
+
+		Storage::disk(StorageDiskType::S3->value)->writeStream(
+			$photo->live_photo_short_path,
+			Storage::disk(StorageDiskType::LOCAL->value)->readStream($photo->live_photo_short_path)
+		);
+
+		Storage::disk(StorageDiskType::LOCAL->value)->delete($photo->live_photo_short_path);
+	}
+}

app/Legacy/Actions/Photo/Strategies/AddStandaloneStrategy.php

@@ -3,6 +3,7 @@
 namespace App\Legacy\Actions\Photo\Strategies;
 
 use App\Actions\Diagnostics\Pipes\Checks\BasicPermissionCheck;
+use App\Assets\Features;
 use App\Contracts\Exceptions\LycheeException;
 use App\Contracts\Image\ImageHandlerInterface;
 use App\Contracts\Image\StreamStats;
@@ -22,8 +23,10 @@
 use App\Image\Handlers\ImageHandler;
 use App\Image\Handlers\VideoHandler;
 use App\Image\StreamStat;
+use App\Jobs\UploadSizeVariantToS3Job;
 use App\Models\Configs;
 use App\Models\Photo;
+use App\Models\SizeVariant;
 use Illuminate\Contracts\Container\BindingResolutionException;
 
 class AddStandaloneStrategy extends AbstractAddStrategy
@@ -143,6 +146,7 @@ public function do(): Photo
 			// we must move the preliminary extracted video file next to the
 			// final target file
 			if ($tmpVideoFile !== null) {
+				// @TODO S3 How should live videos be handled?
 				$videoTargetPath =
 					pathinfo($targetFile->getRelativePath(), PATHINFO_DIRNAME) .
 					'/' .
@@ -169,7 +173,7 @@ public function do(): Photo
 			$imageDim = $this->sourceImage?->isLoaded() ?
 				$this->sourceImage->getDimensions() :
 				new ImageDimension($this->parameters->exifInfo->width, $this->parameters->exifInfo->height);
-			$this->photo->size_variants->create(
+			$originalVariant = $this->photo->size_variants->create(
 				SizeVariantType::ORIGINAL,
 				$targetFile->getRelativePath(),
 				$imageDim,
@@ -193,7 +197,20 @@ public function do(): Photo
 				/** @var SizeVariantFactory $sizeVariantFactory */
 				$sizeVariantFactory = resolve(SizeVariantFactory::class);
 				$sizeVariantFactory->init($this->photo, $this->sourceImage, $this->namingStrategy);
-				$sizeVariantFactory->createSizeVariants();
+				$variants = $sizeVariantFactory->createSizeVariants();
+				$variants->push($originalVariant);
+
+				if (Features::active('use-s3')) {
+					// If enabled, upload all size variants to the remote bucket and delete the local files after that
+					$variants->each(function (SizeVariant $variant) {
+						if (Configs::getValueAsBool('use_job_queues')) {
+							UploadSizeVariantToS3Job::dispatch($variant);
+						} else {
+							$job = new UploadSizeVariantToS3Job($variant);
+							$job->handle();
+						}
+					});
+				}
 			} catch (\Throwable $t) {
 				// Don't re-throw the exception, because we do not want the
 				// import to fail completely only due to missing size variants.

routes/api.php

@@ -45,7 +45,7 @@
 Route::get('/Album::getArchive', [AlbumController::class, 'getArchive'])
 	->name('download')
 	->withoutMiddleware(['content_type:json', 'accept_content_type:json'])
-	->middleware(['local_storage', 'accept_content_type:any']);
+	->middleware(['accept_content_type:any']);
 Route::post('/Album::setTrack', [AlbumController::class, 'setTrack'])
 	->withoutMiddleware(['content_type:json'])
 	->middleware(['content_type:multipart']);
@@ -86,7 +86,7 @@
 Route::get('/Photo::getArchive', [PhotoController::class, 'getArchive'])
 	->name('photo_download')
 	->withoutMiddleware(['content_type:json', 'accept_content_type:json'])
-	->middleware(['local_storage', 'accept_content_type:any']);
+	->middleware(['accept_content_type:any']);
 
 /**
  * SEARCH.