Merge pull request #1096 from rommapp/upload-progress-bars

Upload progress bars

.trunk/trunk.yaml

@@ -2,7 +2,7 @@
 # To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml
 version: 0.1
 cli:
-  version: 1.22.2
+  version: 1.22.3
 # Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins)
 plugins:
   sources:
@@ -19,24 +19,24 @@ runtimes:
 lint:
   enabled:
     - markdownlint@0.41.0
-    - eslint@9.8.0
+    - eslint@9.9.0
     - actionlint@1.7.1
     - bandit@1.7.9
     - black@24.8.0
-    - checkov@3.2.219
+    - checkov@3.2.228
     - git-diff-check
     - isort@5.13.2
     - mypy@1.11.1
     - osv-scanner@1.8.3
     - oxipng@9.1.2
     - prettier@3.3.3
-    - ruff@0.5.7
+    - ruff@0.6.0
     - shellcheck@0.10.0
     - shfmt@3.6.0
     - svgo@3.3.2
     - taplo@0.9.3
     - trivy@0.54.1
-    - trufflehog@3.81.7
+    - trufflehog@3.81.9
     - yamllint@1.35.1
   ignore:
     - linters: [ALL]

backend/endpoints/responses/rom.py

@@ -186,11 +186,6 @@ class UserNotesSchema(TypedDict):
     note_raw_markdown: str
 
 
-class AddRomsResponse(TypedDict):
-    uploaded_roms: list[str]
-    skipped_roms: list[str]
-
-
 class CustomStreamingResponse(StreamingResponse):
     def __init__(self, *args, **kwargs) -> None:
         self.emit_body = kwargs.pop("emit_body", None)

backend/endpoints/rom.py

@@ -14,83 +14,82 @@
 from decorators.auth import protected_route
 from endpoints.responses import MessageResponse
 from endpoints.responses.rom import (
-    AddRomsResponse,
     CustomStreamingResponse,
     DetailedRomSchema,
     RomUserSchema,
     SimpleRomSchema,
 )
 from exceptions.endpoint_exceptions import RomNotFoundInDatabaseException
 from exceptions.fs_exceptions import RomAlreadyExistsException
-from fastapi import File, HTTPException, Query, Request, UploadFile, status
+from fastapi import HTTPException, Query, Request, UploadFile, status
 from fastapi.responses import Response
 from handler.database import db_platform_handler, db_rom_handler
 from handler.filesystem import fs_resource_handler, fs_rom_handler
 from handler.filesystem.base_handler import CoverSize
 from handler.metadata import meta_igdb_handler, meta_moby_handler
 from logger.logger import log
+from starlette.requests import ClientDisconnect
 from stream_zip import NO_COMPRESSION_64, ZIP_AUTO, AsyncMemberFile, async_stream_zip
+from streaming_form_data import StreamingFormDataParser
+from streaming_form_data.targets import FileTarget, NullTarget
 from utils.router import APIRouter
 
 router = APIRouter()
 
 
 @protected_route(router.post, "/roms", ["roms.write"])
-async def add_roms(
-    request: Request,
-    platform_id: int,
-    roms: list[UploadFile] = File(...),  # noqa: B008
-) -> AddRomsResponse:
-    """Upload roms endpoint (one or more at the same time)
+async def add_rom(request: Request):
+    """Upload single rom endpoint
 
     Args:
         request (Request): Fastapi Request object
-        platform_slug (str): Slug of the platform where to upload the roms
-        roms (list[UploadFile], optional): List of files to upload. Defaults to File(...).
 
     Raises:
         HTTPException: No files were uploaded
-
-    Returns:
-        AddRomsResponse: Standard message response
     """
-
-    platform_fs_slug = db_platform_handler.get_platform(platform_id).fs_slug
-    log.info(f"Uploading roms to {platform_fs_slug}")
-    if roms is None:
-        log.error("No roms were uploaded")
+    platform_id = request.headers.get("x-upload-platform")
+    filename = request.headers.get("x-upload-filename")
+    if not platform_id or not filename:
         raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail="No roms were uploaded",
-        )
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="No platform ID or filename provided",
+        ) from None
 
+    platform_fs_slug = db_platform_handler.get_platform(int(platform_id)).fs_slug
     roms_path = fs_rom_handler.build_upload_file_path(platform_fs_slug)
+    log.info(f"Uploading file to {platform_fs_slug}")
 
-    uploaded_roms = []
-    skipped_roms = []
-
-    for rom in roms:
-        if fs_rom_handler.file_exists(roms_path, rom.filename):
-            log.warning(f" - Skipping {rom.filename} since the file already exists")
-            skipped_roms.append(rom.filename)
-            continue
+    file_location = Path(f"{roms_path}/{filename}")
+    parser = StreamingFormDataParser(headers=request.headers)
+    parser.register("x-upload-platform", NullTarget())
+    parser.register(filename, FileTarget(str(file_location)))
 
-        log.info(f" - Uploading {rom.filename}")
-        file_location = f"{roms_path}/{rom.filename}"
+    if await file_location.exists():
+        log.warning(f" - Skipping {filename} since the file already exists")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"File {filename} already exists",
+        ) from None
 
-        async with await open_file(file_location, "wb+") as f:
-            while True:
-                chunk = rom.file.read(8192)
-                if not chunk:
-                    break
-                await f.write(chunk)
+    async def cleanup_partial_file():
+        if await file_location.exists():
+            await file_location.unlink()
 
-        uploaded_roms.append(rom.filename)
+    try:
+        async for chunk in request.stream():
+            parser.data_received(chunk)
+    except ClientDisconnect:
+        log.error("Client disconnected during upload")
+        await cleanup_partial_file()
+    except Exception as exc:
+        log.error("Error uploading files", exc_info=exc)
+        await cleanup_partial_file()
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="There was an error uploading the file(s)",
+        ) from exc
 
-    return {
-        "uploaded_roms": uploaded_roms,
-        "skipped_roms": skipped_roms,
-    }
+    return Response(status_code=status.HTTP_201_CREATED)
 
 
 @protected_route(router.get, "/roms", ["roms.read"])

frontend/src/App.vue

@@ -1,5 +1,6 @@
 <script setup lang="ts">
 import Notification from "@/components/common/Notification.vue";
+import UploadInProgress from "@/components/common/UploadInProgress.vue";
 import api from "@/services/api/index";
 import userApi from "@/services/api/user";
 import storeAuth from "@/stores/auth";
@@ -40,7 +41,7 @@ onBeforeMount(() => {
   <v-app>
     <v-main>
       <notification />
-      <!-- <notification-stack /> -->
+      <upload-in-progress />
       <router-view />
     </v-main>
   </v-app>

frontend/src/components/common/Game/Card/Base.vue

@@ -9,7 +9,6 @@ import storeDownload from "@/stores/download";
 import storeGalleryView from "@/stores/galleryView";
 import storeRoms from "@/stores/roms";
 import { type SimpleRom } from "@/stores/roms.js";
-import { storeToRefs } from "pinia";
 import { onMounted, ref } from "vue";
 import { useTheme } from "vuetify";
 

frontend/src/components/common/Game/Dialog/UploadRom.vue

@@ -6,6 +6,7 @@ import romApi from "@/services/api/rom";
 import socket from "@/services/socket";
 import storeHeartbeat from "@/stores/heartbeat";
 import { type Platform } from "@/stores/platforms";
+import storeUpload from "@/stores/upload";
 import storeScanning from "@/stores/scanning";
 import type { Events } from "@/types/emitter";
 import { formatBytes } from "@/utils";
@@ -16,11 +17,12 @@ import { useDisplay } from "vuetify";
 // Props
 const { xs, mdAndUp, smAndUp } = useDisplay();
 const show = ref(false);
-const romsToUpload = ref<File[]>([]);
+const filesToUpload = ref<File[]>([]);
 const scanningStore = storeScanning();
 const selectedPlatform = ref<Platform | null>(null);
 const supportedPlatforms = ref<Platform[]>();
 const heartbeat = storeHeartbeat();
+const uploadStore = storeUpload();
 const HEADERS = [
   {
     title: "Name",
@@ -64,7 +66,6 @@ emitter?.on("showUploadRomDialog", (platformWhereUpload) => {
 async function uploadRoms() {
   if (!selectedPlatform.value) return;
   show.value = false;
-  scanningStore.set(true);
 
   if (selectedPlatform.value.id == -1) {
     await platformApi
@@ -93,36 +94,38 @@ async function uploadRoms() {
   }
 
   const platformId = selectedPlatform.value.id;
-  emitter?.emit("snackbarShow", {
-    msg: `Uploading ${romsToUpload.value.length} roms to ${selectedPlatform.value.name}...`,
-    icon: "mdi-loading mdi-spin",
-    color: "romm-accent-1",
-  });
 
   await romApi
     .uploadRoms({
-      romsToUpload: romsToUpload.value,
+      filesToUpload: filesToUpload.value,
       platformId: platformId,
     })
-    .then(({ data }) => {
-      const { uploaded_roms, skipped_roms } = data;
+    .then((responses: PromiseSettledResult<unknown>[]) => {
+      uploadStore.clear();
+
+      const successfulUploads = responses.filter(
+        (d) => d.status == "fulfilled"
+      );
+      const failedUploads = responses.filter((d) => d.status == "rejected");
 
-      if (uploaded_roms.length == 0) {
+      if (successfulUploads.length == 0) {
         return emitter?.emit("snackbarShow", {
           msg: `All files skipped, nothing to upload.`,
           icon: "mdi-close-circle",
           color: "orange",
-          timeout: 2000,
+          timeout: 5000,
         });
       }
 
       emitter?.emit("snackbarShow", {
-        msg: `${uploaded_roms.length} files uploaded successfully (and ${skipped_roms.length} skipped). Starting scan...`,
+        msg: `${successfulUploads.length} files uploaded successfully (and ${failedUploads.length} skipped/failed). Starting scan...`,
         icon: "mdi-check-bold",
         color: "green",
-        timeout: 2000,
+        timeout: 3000,
       });
 
+      scanningStore.set(true);
+
       if (!socket.connected) socket.connect();
       setTimeout(() => {
         socket.emit("scan", {
@@ -142,7 +145,7 @@ async function uploadRoms() {
         timeout: 4000,
       });
     });
-  romsToUpload.value = [];
+  filesToUpload.value = [];
   selectedPlatform.value = null;
 }
 
@@ -152,17 +155,19 @@ function triggerFileInput() {
 }
 
 function removeRomFromList(romName: string) {
-  romsToUpload.value = romsToUpload.value.filter((rom) => rom.name !== romName);
+  filesToUpload.value = filesToUpload.value.filter(
+    (rom) => rom.name !== romName
+  );
 }
 
 function closeDialog() {
   show.value = false;
-  romsToUpload.value = [];
+  filesToUpload.value = [];
   selectedPlatform.value = null;
 }
 
 function updateDataTablePages() {
-  pageCount.value = Math.ceil(romsToUpload.value.length / itemsPerPage.value);
+  pageCount.value = Math.ceil(filesToUpload.value.length / itemsPerPage.value);
 }
 watch(itemsPerPage, async () => {
   updateDataTablePages();
@@ -234,7 +239,7 @@ watch(itemsPerPage, async () => {
           </v-btn>
           <v-file-input
             id="file-input"
-            v-model="romsToUpload"
+            v-model="filesToUpload"
             @update:model-value="updateDataTablePages"
             class="file-input"
             multiple
@@ -245,9 +250,9 @@ watch(itemsPerPage, async () => {
     </template>
     <template #content>
       <v-data-table
-        v-if="romsToUpload.length > 0"
+        v-if="filesToUpload.length > 0"
         :item-value="(item) => item.name"
-        :items="romsToUpload"
+        :items="filesToUpload"
         :width="mdAndUp ? '60vw' : '95vw'"
         :items-per-page="itemsPerPage"
         :items-per-page-options="PER_PAGE_OPTIONS"
@@ -316,9 +321,9 @@ watch(itemsPerPage, async () => {
           <v-btn class="bg-terciary" @click="closeDialog"> Cancel </v-btn>
           <v-btn
             class="bg-terciary text-romm-green"
-            :disabled="romsToUpload.length == 0 || selectedPlatform == null"
+            :disabled="filesToUpload.length == 0 || selectedPlatform == null"
             :variant="
-              romsToUpload.length == 0 || selectedPlatform == null
+              filesToUpload.length == 0 || selectedPlatform == null
                 ? 'plain'
                 : 'flat'
             "

frontend/src/components/common/Notification.vue

@@ -17,7 +17,6 @@ emitter?.on("snackbarShow", (snackbar: SnackbarStatus) => {
   show.value = true;
   snackbarStatus.value = snackbar;
   snackbarStatus.value.id = notificationStore.notifications.length + 1;
-  // notificationStore.add(snackbarStatus.value);
 });
 
 function closeDialog() {