Calculate hash values when scanning roms

backend/endpoints/responses/rom.py

@@ -10,7 +10,7 @@
 from handler.metadata.igdb_handler import IGDBMetadata
 from handler.metadata.moby_handler import MobyMetadata
 from pydantic import BaseModel, computed_field, Field
-from models.rom import Rom
+from models.rom import Rom, RomFile
 
 
 SORT_COMPARE_REGEX = r"^([Tt]he|[Aa]|[Aa]nd)\s"
@@ -92,7 +92,7 @@ class RomSchema(BaseModel):
     tags: list[str]
 
     multi: bool
-    files: list[str]
+    files: list[RomFile]
     full_path: str
 
     class Config:

backend/endpoints/rom.py

@@ -25,6 +25,7 @@
 from logger.logger import log
 from stream_zip import ZIP_AUTO, stream_zip  # type: ignore[import]
 from urllib.parse import quote
+
 router = APIRouter()
 
 
@@ -154,7 +155,7 @@ def head_rom_content(request: Request, id: int, file_name: str):
     rom_path = f"{LIBRARY_BASE_PATH}/{rom.full_path}"
 
     return FileResponse(
-        path=rom_path if not rom.multi else f"{rom_path}/{rom.files[0]}",
+        path=rom_path if not rom.multi else f'{rom_path}/{rom.files[0]["filename"]}',
         filename=file_name,
         headers={
             "Content-Disposition": f'attachment; filename="{quote(rom.name)}.zip"',
@@ -187,7 +188,7 @@ def get_rom_content(
 
     rom = db_rom_handler.get_roms(id)
     rom_path = f"{LIBRARY_BASE_PATH}/{rom.full_path}"
-    files_to_download = files or rom.files
+    files_to_download = files or [r["filename"] for r in rom.files]
 
     if not rom.multi:
         return FileResponse(path=rom_path, filename=rom.file_name)
@@ -237,7 +238,9 @@ def contents(f):
     return CustomStreamingResponse(
         zipped_chunks,
         media_type="application/zip",
-        headers={"Content-Disposition": f'attachment; filename="{quote(file_name)}.zip"'},
+        headers={
+            "Content-Disposition": f'attachment; filename="{quote(file_name)}.zip"'
+        },
         emit_body={"id": rom.id},
     )
 

backend/endpoints/sockets/scan.py

@@ -224,6 +224,12 @@ async def scan_platforms(
                             **RomSchema.model_validate(rom).model_dump(),
                         },
                     )
+                elif rom:
+                    # Just to update the filesystem data
+                    rom.file_name = fs_rom["file_name"]
+                    rom.multi = fs_rom["multi"]
+                    rom.files = fs_rom["files"]
+                    db_rom_handler.add_rom(rom)
 
             db_rom_handler.purge_roms(
                 platform.id, [rom["file_name"] for rom in fs_roms]

backend/handler/filesystem/roms_handler.py

@@ -1,7 +1,9 @@
 import os
 import re
-from pathlib import Path
+import binascii
+import hashlib
 import shutil
+from pathlib import Path
 
 from config import LIBRARY_BASE_PATH
 from config.config_manager import config_manager as cm
@@ -81,12 +83,39 @@ def _exclude_multi_roms(self, roms) -> list[str]:
 
         return [f for f in roms if f not in filtered_files]
 
+    def _calculate_rom_size(self, data: bytes) -> int:
+        return {
+            "crc_hash": (binascii.crc32(data) & 0xFFFFFFFF)
+            .to_bytes(4, byteorder="big")
+            .hex(),
+            "md5_hash": hashlib.md5(data).hexdigest(),
+            "sha1_hash": hashlib.sha1(data).hexdigest(),
+        }
+
     def get_rom_files(self, rom: str, roms_path: str) -> list[str]:
         rom_files: list = []
 
-        for path, _, files in os.walk(f"{roms_path}/{rom}"):
-            for f in self._exclude_files(files, "multi_parts"):
-                rom_files.append(f"{Path(path, f)}".replace(f"{roms_path}/{rom}/", ""))
+        # Check if rom is a multi-part rom
+        if os.path.isdir(f"{roms_path}/{rom}"):
+            multi_files = os.listdir(f"{roms_path}/{rom}")
+            for file in multi_files:
+                with open(Path(roms_path, rom, file), "rb") as f:
+                    data = f.read()
+                    rom_files.append(
+                        {
+                            "filename": file,
+                            **self._calculate_rom_size(data),
+                        }
+                    )
+        else:
+            with open(Path(roms_path, rom), "rb") as f:
+                data = f.read()
+                rom_files.append(
+                    {
+                        "filename": rom,
+                        **self._calculate_rom_size(data),
+                    }
+                )
 
         return rom_files
 

backend/handler/scan_handler.py

@@ -26,6 +26,7 @@ class ScanType(Enum):
     UNIDENTIFIED = "unidentified"
     PARTIAL = "partial"
     COMPLETE = "complete"
+    NO_SCAN = "no_scan"
 
 
 def _get_main_platform_igdb_id(platform: Platform):

backend/handler/tests/test_fastapi.py

@@ -25,12 +25,19 @@ def test_scan_platform():
 @pytest.mark.vcr
 async def test_scan_rom():
     platform = Platform(fs_slug="n64", igdb_id=4)
+    files = [{
+        "file_name": "Paper Mario (USA).z64",
+        "crc_hash": "9d0d1c6e",
+        "md5_hash": "f1b7f9e4f4d0e0b7b9faa1b1f2f8e4e9",
+        "sha1_hash": "c3c7f9f3d1d0e0b7b9faa1b1f2f8e4e9",
+    }]
+
     rom = await scan_rom(
         platform,
         {
             "file_name": "Paper Mario (USA).z64",
             "multi": False,
-            "files": ["Paper Mario (USA).z64"],
+            "files": files,
         },
         ScanType.QUICK,
     )
@@ -40,6 +47,6 @@ async def test_scan_rom():
     assert rom.name == "Paper Mario"
     assert rom.igdb_id == 3340
     assert rom.file_size_bytes == 1024
-    assert rom.files == ["Paper Mario (USA).z64"]
+    assert rom.files == files
     assert rom.tags == []
     assert not rom.multi

backend/main.py

@@ -3,7 +3,13 @@
 
 import alembic.config
 import uvicorn
-from config import DEV_HOST, DEV_PORT, ROMM_AUTH_SECRET_KEY, DISABLE_CSRF_PROTECTION
+from config import (
+    DEV_HOST,
+    DEV_PORT,
+    ROMM_AUTH_SECRET_KEY,
+    DISABLE_CSRF_PROTECTION,
+    SCAN_TIMEOUT,
+)
 from contextlib import asynccontextmanager
 from endpoints import (
     auth,
@@ -22,7 +28,7 @@
     feeds,
     firmware,
 )
-import endpoints.sockets.scan  # noqa
+from endpoints.sockets.scan import scan_platforms
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi_pagination import add_pagination
@@ -32,6 +38,8 @@
 from handler.auth.base_handler import ALGORITHM
 from handler.auth.hybrid_auth import HybridAuthBackend
 from handler.auth.middleware import CustomCSRFMiddleware, SessionMiddleware
+from handler.redis_handler import high_prio_queue
+from handler.scan_handler import ScanType
 from starlette.middleware.authentication import AuthenticationMiddleware
 from utils import get_version
 
@@ -103,5 +111,10 @@ async def lifespan(app: FastAPI):
     # Run migrations
     alembic.config.main(argv=["upgrade", "head"])
 
+    # Run a no-scan in the background on startup
+    high_prio_queue.enqueue(
+        scan_platforms, [], ScanType.NO_SCAN, [], [], job_timeout=SCAN_TIMEOUT
+    )
+
     # Run application
     uvicorn.run("main:app", host=DEV_HOST, port=DEV_PORT, reload=True)

backend/models/rom.py

@@ -22,6 +22,14 @@
     or_,
 )
 from sqlalchemy.orm import Mapped, relationship
+from typing_extensions import TypedDict
+
+
+class RomFile(TypedDict):
+    filename: str
+    crc_hash: str
+    md5_hash: str
+    sha1_hash: str
 
 
 class Rom(BaseModel):
@@ -61,7 +69,7 @@ class Rom(BaseModel):
     )
 
     multi: bool = Column(Boolean, default=False)
-    files: JSON = Column(JSON, default=[])
+    files: list[RomFile] = Column(JSON, default=[])
 
     platform_id = Column(
         Integer(),

frontend/src/components/Details/ActionBar.vue

@@ -32,7 +32,7 @@ async function copyDownloadLink(rom: DetailedRom) {
     encodeURI(
       getDownloadLink({
         rom,
-        files: downloadStore.filesToDownloadMultiFileRom,
+        files: downloadStore.filesToDownload,
       })
     );
   if (navigator.clipboard && window.isSecureContext) {
@@ -56,7 +56,7 @@ async function copyDownloadLink(rom: DetailedRom) {
         @click="
           romApi.downloadRom({
             rom,
-            files: downloadStore.filesToDownloadMultiFileRom,
+            files: downloadStore.filesToDownload,
           })
         "
         :disabled="downloadStore.value.includes(rom.id)"

frontend/src/components/Details/Info/FileInfo.vue

@@ -37,8 +37,8 @@ const downloadStore = storeDownload();
       <v-select
         :label="rom.file_name"
         item-title="file_name"
-        v-model="downloadStore.filesToDownloadMultiFileRom"
-        :items="rom.files"
+        v-model="downloadStore.filesToDownload"
+        :items="rom.files.map(f => f.filename)"
         class="my-2"
         density="compact"
         variant="outlined"

frontend/src/stores/download.ts

@@ -3,7 +3,7 @@ import { defineStore } from "pinia";
 export default defineStore("download", {
   state: () => ({
     value: [] as number[],
-    filesToDownloadMultiFileRom: [] as string[],
+    filesToDownload: [] as string[],
   }),
 
   actions: {
@@ -15,7 +15,7 @@ export default defineStore("download", {
     },
     clear() {
       this.value = [] as number[]
-      this.filesToDownloadMultiFileRom = [] as string[]
+      this.filesToDownload = [] as string[]
     }
   },
 });