Feature: CSO/ZSO/DAX support

.gitignore

@@ -124,7 +124,9 @@ site/
 *.cd2
 *.chd
 *.col
+*.cso
 *.cue
+*.dax
 *.dvd
 *.gb
 *.gba
@@ -159,6 +161,7 @@ site/
 *.zim
 *.z64
 *.zip
+*.zso
 
 # ROM pack excess
 *.bmp

docs/alternatives.md

@@ -18,7 +18,8 @@ There are a few different popular ROM managers that have similar features:
 | DATs: fixdat creation                    | ✅ [Fixdat docs](dats/fixdats.md)                                                                 | ✅                                                             | ✅                                                           | ❌                                          |
 | DATs: combine multiple                   | ✅                                                                                                | ❌                                                             | ✅                                                           | ❌                                          |
 | Archives: extraction formats             | ✅ many formats ([reading archives docs](input/reading-archives.md))                              | ✅ `.zip`, `.7z`, `.rar`                                       | ⚠️ `.zip`, `.7z`                                            | ⚠️ `.zip`, `.7z`                           |
-| Archives: `.chd` support                 | ⚠️ via chdman (bundled)                                                                          | ⚠️ via chdman                                                 | ✅ v1-5 natively                                             | ⚠️ v1-4 natively                           |
+| Archives: `.chd` support                 | ⚠️ via `chdman` (bundled)                                                                        | ⚠️ via `chdman`                                               | ✅ v1-5 natively                                             | ⚠️ v1-4 natively                           |
+| Archives: `.cso` & `.zso` support        | ⚠️ via `maxcso` (bundled)                                                                        | ❌                                                             | ❌                                                           | ❌                                          |
 | Archives: `.nkit.iso` support            | ⚠️ matching but no extraction [GameCube docs](usage/console/gamecube.md#nkit)                    | ❌                                                             | ❌                                                           | ❌                                          |
 | Archives: creation formats               | ❌ `.zip` only by design ([writing archives docs](output/writing-archives.md))                    | ✅ `.zip`, `.7z`, `.rar`                                       | ⚠️ `.zip` (TorrentZip), `.7z`                               | ⚠️ `.zip`, `.7z`                           |
 | Archives: automatic extension correction | ✅                                                                                                | ❌                                                             | ❌                                                           | ❌                                          |

docs/input/reading-archives.md

@@ -7,18 +7,19 @@ Igir supports scanning the contents of archives for ROMs, DATs, and ROM patches.
 Igir supports most common archive formats:
 
 | Extension                                                        | Contains file CRC32s | Igir can extract without a third-party binary | Igir can checksum without temporary files |
-|------------------------------------------------------------------|----------------------|-------------------------------------------------|---------------------------------------------|
-| `.7z`                                                            | ✅                    | ❌ `7za`                                         | ❌                                           |
-| `.chd`                                                           | ❌ SHA1               | ❌ `chdman`                                      | ❌                                           |
-| `.gz`, `.gzip`                                                   | ❌ CRC16              | ❌ `7za`                                         | ❌                                           |
-| `.nkit.iso` ([GameCube docs](../usage/console/gamecube.md#nkit)) | ✅                    | ❌ no extraction support                         | ✅                                           |
-| `.rar`                                                           | ✅                    | ✅                                               | ❌                                           |
-| `.tar`                                                           | ❌                    | ✅                                               | ✅ ≤64MiB                                    |
-| `.tar.gz`, `.tgz`                                                | ❌                    | ✅                                               | ✅ ≤64MiB                                    |
-| `.z01`                                                           | ✅                    | ❌ `7za`                                         | ❌                                           |
-| `.zip` (including zip64)                                         | ✅                    | ✅                                               | ✅ ≤64MiB                                    |
-| `.zip.001`                                                       | ✅                    | ❌ `7za`                                         | ❌                                           |
-| `.zipx`                                                          | ✅                    | ❌ `7za`                                         | ❌                                           |
+|------------------------------------------------------------------|----------------------|-----------------------------------------------|-------------------------------------------|
+| `.7z`                                                            | ✅                    | ❌ `7za`                                       | ❌                                         |
+| `.chd`                                                           | ❌ SHA1               | ❌ `chdman`                                    | ❌                                         |
+| `.cso`, `.zso`, `.dax`                                           | ❌                    | ❌ `maxcso`                                    | ⚠️ CRC32 only                             |
+| `.gz`, `.gzip`                                                   | ❌ CRC16              | ❌ `7za`                                       | ❌                                         |
+| `.nkit.iso` ([GameCube docs](../usage/console/gamecube.md#nkit)) | ✅                    | ❌ no extraction support                       | ✅                                         |
+| `.rar`                                                           | ✅                    | ✅                                             | ❌                                         |
+| `.tar`                                                           | ❌                    | ✅                                             | ✅ ≤64MiB                                  |
+| `.tar.gz`, `.tgz`                                                | ❌                    | ✅                                             | ✅ ≤64MiB                                  |
+| `.z01`                                                           | ✅                    | ❌ `7za`                                       | ❌                                         |
+| `.zip` (including zip64)                                         | ✅                    | ✅                                             | ✅ ≤64MiB                                  |
+| `.zip.001`                                                       | ✅                    | ❌ `7za`                                       | ❌                                         |
+| `.zipx`                                                          | ✅                    | ❌ `7za`                                       | ❌                                         |
 
 **You should prefer archive formats that have CRC32 checksum information for each file.**
 

package.json

@@ -86,6 +86,7 @@
     "graceful-fs": "4.2.11",
     "is-admin": "4.0.0",
     "junk": "4.0.1",
+    "maxcso": "0.1130.5",
     "micromatch": "4.0.7",
     "moment": "2.30.1",
     "node-disk-info": "1.3.0",

package.ts

@@ -42,7 +42,7 @@
   return results;
 };
 
 (async (): Promise<void> => {
   const logger = new Logger(LogLevel.TRACE);
 
   const argv = await yargs(process.argv.slice(2))
@@ -89,6 +89,9 @@
     // Only include the exact chdman bin we need
     { exclude: 'node_modules/{**/,}chdman/bin/*/*/chdman*' },
     { include: `node_modules/{**/,}chdman/bin/${process.platform}/${process.arch}/chdman*` },
+    // Only include the exact maxcso bin we need
+    { exclude: 'node_modules/{**/,}maxcso/bin/*/*/maxcso*' },
+    { include: `node_modules/{**/,}maxcso/bin/${process.platform}/${process.arch}/maxcso*` },
   ]));
   const includeSize = (await Promise.all([...include].map(async (file) => {
     if (await FsPoly.isDirectory(file)) {

src/modules/candidateGenerator.ts

@@ -318,16 +318,15 @@ export default class CandidateGenerator extends Module {
     // Filter to the Archives that contain every ROM in this Game
     const archivesWithEveryRom = [...inputArchivesToRoms.entries()]
       .filter(([archive, roms]) => {
-        if (roms.map((rom) => rom.hashCode()).join(',') === game.getRoms().map((rom) => rom.hashCode()).join(',')) {
+        if (roms.map((rom) => rom.hashCode()).join(',') === gameRoms.map((rom) => rom.hashCode()).join(',')) {
           return true;
         }
         // If there is a CHD with every .bin file, and we're raw-copying it, then assume its .cue
         // file is accurate
         return archive instanceof Chd
-          && !game.getRoms().some((rom) => this.options.shouldZipRom(rom))
-          && !game.getRoms().some((rom) => this.options.shouldExtractRom(rom))
-          && CandidateGenerator.onlyCueFilesMissingFromChd(game, roms)
-          && this.options.getAllowExcessSets();
+          && !gameRoms.some((rom) => this.options.shouldZipRom(rom))
+          && !gameRoms.some((rom) => this.options.shouldExtractRom(rom))
+          && CandidateGenerator.onlyCueFilesMissingFromChd(game, roms);
       })
       .map(([archive]) => archive);
 
@@ -358,7 +357,7 @@ export default class CandidateGenerator extends Module {
     if (
       // The Game has zero or one ROM, therefore, we don't really care where the file comes from,
       //  and we should respect any previous sorting of the input files
-      game.getRoms().length <= 1
+      gameRoms.length <= 1
       // No input archive contains every ROM from this Game
       || archiveWithEveryRom === undefined
       // We're extracting files, therefore, we don't really care where the file comes from, and we
@@ -374,10 +373,19 @@ export default class CandidateGenerator extends Module {
     // For each of this Game's ROMs, find the matching ArchiveEntry from this Archive
     return new Map(romsAndInputFiles.map(([rom, inputFiles]) => {
       this.progressBar.logTrace(`${dat.getNameShort()}: ${game.getName()}: preferring input archive that contains every ROM: ${archiveWithEveryRom.getFilePath()}`);
-      const archiveEntry = inputFiles.find((
-        inputFile,
-      ) => inputFile.getFilePath() === archiveWithEveryRom.getFilePath()) as File;
-      return [rom, archiveEntry];
+      let archiveEntry = inputFiles
+        .find((inputFile) => inputFile.getFilePath() === archiveWithEveryRom.getFilePath());
+
+      if (!archiveEntry
+        && rom.getName().toLowerCase().endsWith('.cue')
+        && archiveWithEveryRom instanceof Chd
+      ) {
+        // We assumed this CHD was fine above, find its .cue file
+        archiveEntry = (filesByPath.get(archiveWithEveryRom.getFilePath()) ?? [])
+          .find((file) => file.getExtractedFilePath().toLowerCase().endsWith('.cue'));
+      }
+
+      return [rom, archiveEntry as File];
     }));
   }
 

src/modules/romIndexer.ts

@@ -3,6 +3,8 @@ import path from 'node:path';
 import ProgressBar, { ProgressBarSymbol } from '../console/progressBar.js';
 import FsPoly from '../polyfill/fsPoly.js';
 import ArchiveEntry from '../types/files/archives/archiveEntry.js';
+import Chd from '../types/files/archives/chd/chd.js';
+import Maxcso from '../types/files/archives/maxcso/maxcso.js';
 import Rar from '../types/files/archives/rar.js';
 import SevenZip from '../types/files/archives/sevenZip/sevenZip.js';
 import Tar from '../types/files/archives/tar.js';
@@ -97,6 +99,10 @@ export default class ROMIndexer extends Module {
       return 3;
     } if (file.getArchive() instanceof SevenZip) {
       return 4;
+    } if (file.getArchive() instanceof Maxcso) {
+      return 5;
+    } if (file.getArchive() instanceof Chd) {
+      return 6;
     }
     return 99;
   }

src/polyfill/fsPoly.ts

@@ -253,11 +253,11 @@ export default class FsPoly {
       }
 
       // Backoff with jitter
-      if (attempt >= 3) {
+      if (attempt >= 5) {
         throw error;
       }
       await new Promise((resolve) => {
-        setTimeout(resolve, Math.random() * (2 ** (attempt - 1) * 100));
+        setTimeout(resolve, Math.random() * (2 ** (attempt - 1) * 10));
       });
 
       // Attempt to resolve Windows' "EBUSY: resource busy or locked"

src/types/files/archives/chd/chdGdiParser.ts

@@ -23,7 +23,7 @@ export default class ChdGdiParser {
     checksumBitmask: number,
   ): Promise<ArchiveEntry<T>[]> {
     const tempDir = await FsPoly.mkdtemp(path.join(Temp.getTempDir(), 'chd-gdi'));
-    const gdiFilePath = path.join(tempDir, 'track.gdi');
+    const gdiFilePath = path.join(tempDir, `${path.parse(archive.getFilePath()).name}.gdi`);
     let binRawFilePaths: string[] = [];
 
     try {

src/types/files/archives/maxcso/cso.ts

@@ -0,0 +1,18 @@
+import Archive from '../archive.js';
+import Maxcso from './maxcso.js';
+
+export default class Cso extends Maxcso {
+  // eslint-disable-next-line class-methods-use-this
+  protected new(filePath: string): Archive {
+    return new Cso(filePath);
+  }
+
+  static getExtensions(): string[] {
+    return ['.cso'];
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  getExtension(): string {
+    return Cso.getExtensions()[0];
+  }
+}

src/types/files/archives/maxcso/dax.ts

@@ -0,0 +1,18 @@
+import Archive from '../archive.js';
+import Maxcso from './maxcso.js';
+
+export default class Dax extends Maxcso {
+  // eslint-disable-next-line class-methods-use-this
+  protected new(filePath: string): Archive {
+    return new Dax(filePath);
+  }
+
+  static getExtensions(): string[] {
+    return ['.dax'];
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  getExtension(): string {
+    return Dax.getExtensions()[0];
+  }
+}

src/types/files/archives/maxcso/maxcso.ts

@@ -0,0 +1,28 @@
+import path from 'node:path';
+
+import maxcso from 'maxcso';
+
+import Archive from '../archive.js';
+import ArchiveEntry from '../archiveEntry.js';
+
+export default abstract class Maxcso extends Archive {
+  async getArchiveEntries(checksumBitmask: number): Promise<ArchiveEntry<Archive>[]> {
+    const entryPath = `${path.parse(this.getFilePath()).name}.iso`;
+    const size = (await maxcso.header(this.getFilePath())).uncompressedSize;
+    const crc32 = await maxcso.uncompressedCrc32(this.getFilePath());
+
+    return [await ArchiveEntry.entryOf({
+      archive: this,
+      entryPath,
+      size: Number(size),
+      crc32,
+    }, checksumBitmask)];
+  }
+
+  async extractEntryToFile(entryPath: string, extractedFilePath: string): Promise<void> {
+    return maxcso.decompress({
+      inputFilename: this.getFilePath(),
+      outputFilename: extractedFilePath,
+    });
+  }
+}

src/types/files/archives/maxcso/zso.ts

@@ -0,0 +1,18 @@
+import Archive from '../archive.js';
+import Maxcso from './maxcso.js';
+
+export default class Zso extends Maxcso {
+  // eslint-disable-next-line class-methods-use-this
+  protected new(filePath: string): Archive {
+    return new Zso(filePath);
+  }
+
+  static getExtensions(): string[] {
+    return ['.zso'];
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  getExtension(): string {
+    return Zso.getExtensions()[0];
+  }
+}

src/types/files/archives/sevenZip/sevenZip.ts

@@ -31,17 +31,23 @@ export default class SevenZip extends Archive {
   async getArchiveEntries(checksumBitmask: number): Promise<ArchiveEntry<Archive>[]> {
     /**
      * WARN(cemmer): even with the above mutex, {@link _7z.list} will still sometimes return no
-     *  entries. Most archives contain at least one file, so assume this is wrong and attempt
-     *  again up to 3 times total.
+     * entries. This seems to happen more on older Node.js versions (v16, v18) and specific OSes
+     * (Linux). Most archives contain at least one file, so assume this is wrong and attempt again
+     * up to 5 times total.
      */
-    for (let attempt = 1; attempt <= 3; attempt += 1) {
+    const maxAttempts = 5;
+    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
       const archiveEntries = await this.getArchiveEntriesNotCached(checksumBitmask);
       if (archiveEntries.length > 0) {
         return archiveEntries;
       }
 
+      // Backoff with jitter
+      if (attempt >= maxAttempts) {
+        break;
+      }
       await new Promise((resolve) => {
-        setTimeout(resolve, Math.random() * (2 ** (attempt - 1) * 100));
+        setTimeout(resolve, Math.random() * (2 ** (attempt - 1) * 10));
       });
     }
 

src/types/files/fileCache.ts

@@ -202,6 +202,12 @@ export default class FileCache {
   }
 
   static async getOrComputeFileHeader(file: File): Promise<ROMHeader | undefined> {
+    if (!this.enabled) {
+      return file.createReadStream(
+        async (stream) => ROMHeader.headerFromFileStream(stream),
+      );
+    }
+
     // NOTE(cemmer): we're explicitly not catching ENOENT errors here, we want it to bubble up
     const stats = await FsPoly.stat(file.getFilePath());
     const cacheKey = this.getCacheKey(file.toString(), ValueType.ROM_HEADER);
@@ -236,6 +242,12 @@ export default class FileCache {
   }
 
   static async getOrComputeFileSignature(file: File): Promise<FileSignature | undefined> {
+    if (!this.enabled) {
+      return file.createReadStream(
+        async (stream) => FileSignature.signatureFromFileStream(stream),
+      );
+    }
+
     // NOTE(cemmer): we're explicitly not catching ENOENT errors here, we want it to bubble up
     const stats = await FsPoly.stat(file.getFilePath());
     const cacheKey = this.getCacheKey(file.toString(), ValueType.FILE_SIGNATURE);