Feature: CHD support (#1006)

.gitattributes

@@ -1,4 +1,4 @@
 # Stop `core.autocrlf true`
-*.lnx binary
-*.nes binary
-*.rom binary
+test/fixtures/roms/**  binary
+*.cue  text eol=lf
+*.gdi  text eol=crlf

.github/workflows/codecov.yml

@@ -35,6 +35,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: volta-cli/action@v4
       - run: npm ci
+      - run: |
+          sudo apt-get update
+          sudo apt-get install -y libsdl2-2.0-0 libsdl2-ttf-2.0-0
 
       # Run test coverage
       - run: npm run test:coverage

.github/workflows/node-test.yml

@@ -82,6 +82,13 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
       - run: npm ci
+      - if: ${{ matrix.os == 'macos' }}
+        run: brew install --overwrite sdl2
+      - if: ${{ matrix.os == 'ubuntu' }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libsdl2-2.0-0 libsdl2-ttf-2.0-0
+
       # Test the source files
       - run: npm run test:unit
 
@@ -102,6 +109,9 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
       - run: npm ci
+      - run: |
+          sudo apt-get update
+          sudo apt-get install -y libsdl2-2.0-0 libsdl2-ttf-2.0-0
       # Test the built files
       - run: npm run build
       - run: ./test/endToEndTest.sh

.gitignore

@@ -122,6 +122,7 @@ site/
 *.bin
 *.cd1
 *.cd2
+*.chd
 *.col
 *.cue
 *.dvd
@@ -144,6 +145,7 @@ site/
 *.pce
 *.pk3
 *.pup
+*.raw
 *.rom
 *.rvz
 *.sfc

docs/alternatives.md

@@ -18,7 +18,7 @@ There are a few different popular ROM managers that have similar features:
 | DATs: fixdat creation                    | ✅ [Fixdat docs](dats/fixdats.md)                                                                 | ✅                                                             | ✅                                                           | ❌                                          |
 | DATs: combine multiple                   | ✅                                                                                                | ❌                                                             | ✅                                                           | ❌                                          |
 | Archives: extraction formats             | ✅ many formats ([reading archives docs](input/reading-archives.md))                              | ✅ `.zip`, `.7z`, `.rar`                                       | ⚠️ `.zip`, `.7z`                                            | ⚠️ `.zip`, `.7z`                           |
-| Archives: `.chd` support                 | ❌                                                                                                | ⚠️ via chdman                                                 | ✅ v1-5 natively                                             | ⚠️ v1-4 natively                           |
+| Archives: `.chd` support                 | ⚠️ via chdman (bundled)                                                                          | ⚠️ via chdman                                                 | ✅ v1-5 natively                                             | ⚠️ v1-4 natively                           |
 | Archives: `.nkit.iso` support            | ⚠️ matching but no extraction [GameCube docs](usage/console/gamecube.md#nkit)                    | ❌                                                             | ❌                                                           | ❌                                          |
 | Archives: creation formats               | ❌ `.zip` only by design ([writing archives docs](output/writing-archives.md))                    | ✅ `.zip`, `.7z`, `.rar`                                       | ⚠️ `.zip` (TorrentZip), `.7z`                               | ⚠️ `.zip`, `.7z`                           |
 | Archives: automatic extension correction | ✅                                                                                                | ❌                                                             | ❌                                                           | ❌                                          |

docs/input/reading-archives.md

@@ -6,18 +6,19 @@
 
 `igir` supports most common archive formats:
 
-| Extension                                                    | Contains file CRC32s | `igir` can extract without a third-party binary | `igir` can checksum without temporary files |
-|--------------------------------------------------------------|----------------------|-------------------------------------------------|---------------------------------------------|
-| `.7z`                                                        | ✅                    | ❌                                               | ❌                                           |
-| `.gz`, `.gzip`                                               | ❌ CRC16              | ❌                                               | ❌                                           |
-| `.nkit` ([GameCube docs](../usage/console/gamecube.md#nkit)) | ✅                    | ❌ no extraction support                         | ✅                                           |
-| `.rar`                                                       | ✅                    | ✅                                               | ❌                                           |
-| `.tar`                                                       | ❌                    | ✅                                               | ✅ ≤64MiB                                    |
-| `.tar.gz`, `.tgz`                                            | ❌                    | ✅                                               | ✅ ≤64MiB                                    |
-| `.z01`                                                       | ✅                    | ❌                                               | ❌                                           |
-| `.zip` (including zip64)                                     | ✅                    | ✅                                               | ✅ ≤64MiB                                    |
-| `.zip.001`                                                   | ✅                    | ❌                                               | ❌                                           |
-| `.zipx`                                                      | ✅                    | ❌                                               | ❌                                           |
+| Extension                                                        | Contains file CRC32s | `igir` can extract without a third-party binary | `igir` can checksum without temporary files |
+|------------------------------------------------------------------|----------------------|-------------------------------------------------|---------------------------------------------|
+| `.7z`                                                            | ✅                    | ❌ `7za`                                         | ❌                                           |
+| `.chd`                                                           | ❌ SHA1               | ❌ `chdman`                                      | ❌                                           |
+| `.gz`, `.gzip`                                                   | ❌ CRC16              | ❌ `7za`                                         | ❌                                           |
+| `.nkit.iso` ([GameCube docs](../usage/console/gamecube.md#nkit)) | ✅                    | ❌ no extraction support                         | ✅                                           |
+| `.rar`                                                           | ✅                    | ✅                                               | ❌                                           |
+| `.tar`                                                           | ❌                    | ✅                                               | ✅ ≤64MiB                                    |
+| `.tar.gz`, `.tgz`                                                | ❌                    | ✅                                               | ✅ ≤64MiB                                    |
+| `.z01`                                                           | ✅                    | ❌ `7za`                                         | ❌                                           |
+| `.zip` (including zip64)                                         | ✅                    | ✅                                               | ✅ ≤64MiB                                    |
+| `.zip.001`                                                       | ✅                    | ❌ `7za`                                         | ❌                                           |
+| `.zipx`                                                          | ✅                    | ❌ `7za`                                         | ❌                                           |
 
 **You should prefer archive formats that have CRC32 checksum information for each file.**
 

jest.config.ts

@@ -1,6 +1,26 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
 import { JestConfigWithTsJest } from 'ts-jest';
 
+// Fix some bad package.json files that don't play well with ts-jest
+[
+  // https://github.com/g-plane/cue/issues/1
+  '@gplane/cue',
+].forEach((moduleName) => {
+  const modulePath = path.join('node_modules', moduleName);
+  const packagePath = path.join(modulePath, 'package.json');
+  const packageJson = JSON.parse(fs.readFileSync(packagePath).toString());
+
+  packageJson.main = packageJson.main
+    ?? packageJson.exports['.'].import;
+  delete packageJson.exports;
+
+  fs.writeFileSync(packagePath, JSON.stringify(packageJson, undefined, 2));
+});
+
 const jestConfig: JestConfigWithTsJest = {
+  preset: 'ts-jest',
   testEnvironment: 'node',
 
   setupFilesAfterEnv: ['jest-extended/all'],

package.json

@@ -70,12 +70,14 @@
   "dependencies": {
     "@fast-csv/format": "5.0.0",
     "@fast-csv/parse": "5.0.0",
+    "@gplane/cue": "0.2.0",
     "@node-rs/crc32": "1.10.3",
     "7zip-min": "1.4.4",
     "archiver": "7.0.1",
     "async": "3.2.5",
     "async-mutex": "0.5.0",
     "chalk": "5.3.0",
+    "chdman": "0.267.2",
     "class-transformer": "0.5.1",
     "cli-progress": "3.12.0",
     "fast-glob": "3.3.2",

package.ts

@@ -86,6 +86,9 @@ const fileFilter = (filters: FileFilter[]): string[] => {
     // Only include the exact 7zip-bin we need
     { exclude: 'node_modules/{**/,}7zip-bin/**/7z*' },
     { include: path7za },
+    // Only include the exact chdman bin we need
+    { exclude: 'node_modules/{**/,}chdman/bin/*/*/chdman*' },
+    { include: `node_modules/{**/,}chdman/bin/${process.platform}/${process.arch}/chdman*` },
   ]));
   const includeSize = (await Promise.all([...include].map(async (file) => {
     if (await FsPoly.isDirectory(file)) {

src/console/progressBar.ts

@@ -23,6 +23,7 @@ export const ProgressBarSymbol = {
   // Candidates
   GENERATING: chalk.cyan('Σ'),
   FILTERING: chalk.cyan('∆'),
+  EXTENSION_CORRECTION: chalk.cyan('.'),
   HASHING: chalk.cyan('#'),
   VALIDATING: chalk.cyan(process.platform === 'win32' ? '?' : '≟'),
   COMBINING_ALL: chalk.cyan(process.platform === 'win32' ? 'U' : '∪'),

src/modules/candidateExtensionCorrector.ts

@@ -7,6 +7,7 @@ import DAT from '../types/dats/dat.js';
 import Parent from '../types/dats/parent.js';
 import ROM from '../types/dats/rom.js';
 import ArchiveEntry from '../types/files/archives/archiveEntry.js';
+import Chd from '../types/files/archives/chd/chd.js';
 import FileSignature from '../types/files/fileSignature.js';
 import Options, { FixExtension } from '../types/options.js';
 import OutputFactory from '../types/outputFactory.js';
@@ -52,7 +53,7 @@ export default class CandidateExtensionCorrector extends Module {
       .filter((romWithFiles) => this.romNeedsCorrecting(romWithFiles))
       .length;
     this.progressBar.logTrace(`${dat.getNameShort()}: correcting ${romsThatNeedCorrecting.toLocaleString()} output file extension${romsThatNeedCorrecting !== 1 ? 's' : ''}`);
-    await this.progressBar.setSymbol(ProgressBarSymbol.HASHING);
+    await this.progressBar.setSymbol(ProgressBarSymbol.EXTENSION_CORRECTION);
     await this.progressBar.reset(romsThatNeedCorrecting);
 
     const correctedParentsToCandidates = await this.correctExtensions(dat, parentsToCandidates);
@@ -62,6 +63,12 @@ export default class CandidateExtensionCorrector extends Module {
   }
 
   private romNeedsCorrecting(romWithFiles: ROMWithFiles): boolean {
+    const inputFile = romWithFiles.getInputFile();
+    if (inputFile instanceof ArchiveEntry && inputFile.getArchive() instanceof Chd) {
+      // Files within CHDs never need extension correction
+      return false;
+    }
+
     return this.options.getFixExtension() === FixExtension.ALWAYS
       || (this.options.getFixExtension() === FixExtension.AUTO && (
         !this.options.usingDats()
@@ -148,21 +155,25 @@ export default class CandidateExtensionCorrector extends Module {
       this.progressBar.logTrace(`${dat.getNameShort()}: ${parent.getName()}: correcting extension for: ${romWithFiles.getInputFile()
         .toString()}`);
 
-      await romWithFiles.getInputFile().createReadStream(async (stream) => {
-        const romSignature = await FileSignature.signatureFromFileStream(stream);
-        if (!romSignature) {
-          // No signature was found, so we can't perform any correction
-          return;
-        }
-
-        // ROM file signature found, use the appropriate extension
-        const { dir, name } = path.parse(correctedRom.getName());
-        const correctedRomName = path.format({
-          dir,
-          name: name + romSignature.getExtension(),
+      try {
+        await romWithFiles.getInputFile().createReadStream(async (stream) => {
+          const romSignature = await FileSignature.signatureFromFileStream(stream);
+          if (!romSignature) {
+            // No signature was found, so we can't perform any correction
+            return;
+          }
+
+          // ROM file signature found, use the appropriate extension
+          const { dir, name } = path.parse(correctedRom.getName());
+          const correctedRomName = path.format({
+            dir,
+            name: name + romSignature.getExtension(),
+          });
+          correctedRom = correctedRom.withName(correctedRomName);
         });
-        correctedRom = correctedRom.withName(correctedRomName);
-      });
+      } catch (error) {
+        this.progressBar.logError(`${dat.getNameShort()}: failed to correct file extension for '${romWithFiles.getInputFile()}': ${error}`);
+      }
 
       this.progressBar.removeWaitingMessage(waitingMessage);
       await this.progressBar.incrementDone();