Read into mem in chunks

backend/endpoints/rom.py

@@ -78,10 +78,7 @@ def add_roms(
         file_location = f"{roms_path}/{rom.filename}"
 
         with open(file_location, "wb+") as f:
-            while True:
-                chunk = rom.file.read(1024)
-                if not chunk:
-                    break
+            while chunk := f.read(8192):
                 f.write(chunk)
 
         uploaded_roms.append(rom.filename)

backend/handler/filesystem/firmware_handler.py

@@ -51,13 +51,20 @@ def get_firmware_file_size(self, firmware_path: str, file_name: str):
 
     def calculate_file_hashes(self, firmware_path: str, file_name: str):
         with open(f"{LIBRARY_BASE_PATH}/{firmware_path}/{file_name}", "rb") as f:
-            data = f.read()
+            crc_c = 0
+            md5_h = hashlib.md5(usedforsecurity=False)
+            sha1_h = hashlib.sha1(usedforsecurity=False)
+
+            # Read in chunks to avoid memory issues
+            while chunk := f.read(8192):
+                md5_h.update(chunk)
+                sha1_h.update(chunk)
+                crc_c = binascii.crc32(chunk, crc_c)
+
             return {
-                "crc_hash": (binascii.crc32(data) & 0xFFFFFFFF)
-                .to_bytes(4, byteorder="big")
-                .hex(),
-                "md5_hash": hashlib.md5(data, usedforsecurity=False).hexdigest(),
-                "sha1_hash": hashlib.sha1(data, usedforsecurity=False).hexdigest(),
+                "crc_hash": (crc_c & 0xFFFFFFFF).to_bytes(4, byteorder="big").hex(),
+                "md5_hash": md5_h.hexdigest(),
+                "sha1_hash": sha1_h.hexdigest(),
             }
 
     def file_exists(self, path: str, file_name: str):

backend/handler/filesystem/roms_handler.py

@@ -86,13 +86,22 @@ def _exclude_multi_roms(self, roms) -> list[str]:
 
         return [f for f in roms if f not in filtered_files]
 
-    def _calculate_rom_size(self, data: bytes) -> int:
+    def _calculate_rom_size(self, file_path: Path) -> dict[str, str]:
+        crc_c = 0
+        md5_h = hashlib.md5(usedforsecurity=False)
+        sha1_h = hashlib.sha1(usedforsecurity=False)
+
+        with open(file_path, "rb") as f:
+            # Read in chunks to avoid memory issues
+            while chunk := f.read(8192):
+                md5_h.update(chunk)
+                sha1_h.update(chunk)
+                crc_c = binascii.crc32(chunk, crc_c)
+
         return {
-            "crc_hash": (binascii.crc32(data) & 0xFFFFFFFF)
-            .to_bytes(4, byteorder="big")
-            .hex(),
-            "md5_hash": hashlib.md5(data).hexdigest(),
-            "sha1_hash": hashlib.sha1(data).hexdigest(),
+            "crc_hash": (crc_c & 0xFFFFFFFF).to_bytes(4, byteorder="big").hex(),
+            "md5_hash": md5_h.hexdigest(),
+            "sha1_hash": sha1_h.hexdigest(),
         }
 
     def get_rom_files(self, rom: str, roms_path: str) -> list[str]:
@@ -102,23 +111,19 @@ def get_rom_files(self, rom: str, roms_path: str) -> list[str]:
         if os.path.isdir(f"{roms_path}/{rom}"):
             multi_files = os.listdir(f"{roms_path}/{rom}")
             for file in multi_files:
-                with open(Path(roms_path, rom, file), "rb") as f:
-                    data = f.read()
-                    rom_files.append(
-                        {
-                            "filename": file,
-                            **self._calculate_rom_size(data),
-                        }
-                    )
-        else:
-            with open(Path(roms_path, rom), "rb") as f:
-                data = f.read()
                 rom_files.append(
                     {
-                        "filename": rom,
-                        **self._calculate_rom_size(data),
+                        "filename": file,
+                        **self._calculate_rom_size(Path(roms_path, rom, file)),
                     }
                 )
+        else:
+            rom_files.append(
+                {
+                    "filename": rom,
+                    **self._calculate_rom_size(Path(roms_path, rom)),
+                }
+            )
 
         return rom_files