Feature: correct bad ROM extensions (#1174)

.gitignore

@@ -118,31 +118,59 @@ site/
 
 # ROMs
 *.7z
+*.a52
 *.bin
+*.cd1
+*.cd2
+*.col
 *.cue
 *.dvd
 *.gb
 *.gba
 *.gbc
 *.gdi
+*.gg
+*.ic1
+*.img
+*.jar
+*.lo
+*.lyx
+*.min
+*.nds
 *.nes
+*.ngp
+*.pce
+*.pk3
+*.pup
 *.rom
 *.sfc
 *.smc
+*.sms
+*.szx
+*.wad
+*.x1
+*.x1t
+*.zim
+*.z64
 *.zip
 
 # ROM pack excess
+*.bmp
 *.csv
+*.db
 *.desktop
+*.fnt
 *.jpg
 *.png
 *.sa1
 *.sav
 *.sg1
+*.sha
 *.sgm
 *.sqlite
 *.torrent
 *.txt
+*.wav
 *.xml
 
 # ROM patches

docs/advanced/internals.md

@@ -17,6 +17,8 @@ Information about the inner workings of `igir`.
    - Input files are matched to ROMs in the DAT (see [matching docs](../roms/matching.md))
    - Patch files are matched to ROMs found (see [patching docs](../roms/patching.md))
    - ROM preferences are applied (`--single`, see [filtering & preference docs](../roms/filtering-preferences.md#preferences-for-1g1r))
+   - ROMs without a potentially bad extension have their extension corrected using its file signature
+   - ROM archives that aren't being extracted have their checksums calculated
    - ROMs are combined (`--zip-dat-name`)
    - ROMs are written to the output directory, if specified (`copy`, `move`, `link`)
      - Written ROMs are tested for accuracy, if specified (`test`)

docs/advanced/logging.md

@@ -31,7 +31,7 @@ There are additional levels of verbosity that can be enabled with the `-v` flag:
   This includes:
 
   - Everything from the `INFO` level above
-  - Files skipped from being copied, zipped, or linked because the output file exists and an `--overwrite` option wasn't provided
+  - Files skipped from being copied, zipped, or linked because the output file exists and an [`--overwrite` option](../output/options.md#overwriting-files) wasn't provided
   - [Fixdat](../dats/fixdats.md) files skipped from being created because all games were found
 
   Usage:

docs/dats/dir2dat.md

@@ -81,6 +81,10 @@ Once DATs have been generated from input files, they are processed the same as a
 
   If your input files are in some kind of standard naming convention (e.g. [No-Intro](https://wiki.no-intro.org/index.php?title=Naming_Convention), [Redump](https://datomatic.no-intro.org/stuff/The%20Official%20No-Intro%20Convention%20(20071030).pdf), or [TOSEC](https://www.tosecdev.org/tosec-naming-convention)) that contains region, language, or other tags, then [ROM filter options](../roms/filtering-preferences.md) can be applied.
 
+- **Filename extensions can be corrected.**
+
+    See [ROM Output Options](../output/options.md#fixing-rom-extensions) for more information.
+
 ## Alternative tools
 
 It is unlikely that any ROM tool, including `igir`, will ever meet every person's exact DAT creation needs.

docs/output/options.md

@@ -0,0 +1,37 @@
+# ROM Writing Options
+
+## Overwriting files
+
+By default, `igir` will _not_ overwrite or delete any files already in the output directory.
+
+To change this behavior, the `--overwrite` option will force overwriting files in the output directory as necessary. Be careful with this option as it can cause unnecessary wear and tear on your hard drives.
+
+The `--overwrite-invalid` option can also overwrite files in the output directory, but _only_ if those files don't match the expected size and checksum. This uses the same logic as the `igir test` command. Combining this option with the [`igir clean` command](./cleaning.md) will result in your output directory being a perfect subset of files contained in your [DATs](../dats/introduction.md).
+
+## Fixing ROM extensions
+
+ROM dumpers don't always do a good job of using the generally accepted filename extension when writing files. In situations where DATs aren't provided, or information in DATs is incomplete, `igir` has some ability to find the correct extension that filenames should have. This is done using [file signatures](https://en.wikipedia.org/wiki/List_of_file_signatures), pieces of data that are common to every file of a certain format.
+
+Here are some examples of common mistakes:
+
+| Incorrect extensions                                                   | Correct extension                    |
+   |------------------------------------------------------------------------|--------------------------------------|
+| `.fc` Nintendo Family Computer<br>`.nez` Nintendo Entertainment System | `.nes` Nintendo Entertainment System |
+| `.sgb` Nintendo Super Game Boy                                         | `.gbc` Nintendo Game Boy Color       |
+| `.bin` Sega Mega Drive / Genesis<br>`.gen` Sega Genesis                | `.md` Sega Mega Drive                |
+
+This correction behavior can be controlled with the following option:
+
+- `--rom-fix-extension never`
+
+  Don't correct any ROM filename extensions. If a DAT doesn't provide a ROM filename, a default name of `<game name>.rom` will be used.
+
+- `--rom-fix-extension auto` (default)
+
+  When not using DATs (no [`--dat <path>` option](../dats/processing.md) was provided), or when a DAT doesn't specify the filename for a ROM, then try to correct the filename extension.
+
+- `--rom-fix-extension always`
+
+  Always try to correct filename extensions, ignoring the information provided by DATs. You likely don't want this option.
+
+See the `igir --help` message for the list of all known file types.

mkdocs.yml

@@ -95,6 +95,7 @@ nav:
       - File Outputs:
           - output/path-options.md
           - output/tokens.md
+          - output/options.md
           - output/writing-archives.md
           - output/reporting.md
           - output/cleaning.md

src/igir.ts

@@ -12,6 +12,7 @@ import Package from './globals/package.js';
 import Temp from './globals/temp.js';
 import CandidateArchiveFileHasher from './modules/candidateArchiveFileHasher.js';
 import CandidateCombiner from './modules/candidateCombiner.js';
+import CandidateExtensionCorrector from './modules/candidateExtensionCorrector.js';
 import CandidateGenerator from './modules/candidateGenerator.js';
 import CandidateMergeSplitValidator from './modules/candidateMergeSplitValidator.js';
 import CandidatePatchGenerator from './modules/candidatePatchGenerator.js';
@@ -388,10 +389,15 @@ export default class Igir {
     const preferredCandidates = await new CandidatePreferer(this.options, progressBar)
       .prefer(dat, patchedCandidates);
 
+    const extensionCorrectedCandidates = await new CandidateExtensionCorrector(
+      this.options,
+      progressBar,
+    ).correct(dat, preferredCandidates);
+
     // Delay calculating checksums for {@link ArchiveFile}s until after {@link CandidatePreferer}
     //  for efficiency
     const hashedCandidates = await new CandidateArchiveFileHasher(this.options, progressBar)
-      .hash(dat, preferredCandidates);
+      .hash(dat, extensionCorrectedCandidates);
 
     const postProcessedCandidates = await new CandidatePostProcessor(this.options, progressBar)
       .process(dat, hashedCandidates);

src/modules/argumentsParser.ts

@@ -10,8 +10,14 @@ import ConsolePoly from '../polyfill/consolePoly.js';
 import ExpectedError from '../types/expectedError.js';
 import { ChecksumBitmask } from '../types/files/fileChecksums.js';
 import ROMHeader from '../types/files/romHeader.js';
+import ROMSignature from '../types/files/romSignature.js';
 import Internationalization from '../types/internationalization.js';
-import Options, { GameSubdirMode, InputChecksumArchivesMode, MergeMode } from '../types/options.js';
+import Options, {
+  GameSubdirMode,
+  InputChecksumArchivesMode,
+  MergeMode,
+  RomFixExtension,
+} from '../types/options.js';
 import PatchFactory from '../types/patches/patchFactory.js';
 
 /**
@@ -68,7 +74,8 @@ export default class ArgumentsParser {
     const groupRomInput = 'ROM input options:';
     const groupDatInput = 'DAT input options:';
     const groupPatchInput = 'Patch input options:';
-    const groupRomOutput = 'ROM output options (processed in order):';
+    const groupRomOutputPath = 'ROM output path options (processed in order):';
+    const groupRomOutput = 'ROM writing options:';
     const groupRomClean = 'clean command options:';
     const groupRomZip = 'zip command options:';
     const groupRomLink = 'link command options:';
@@ -229,7 +236,15 @@ export default class ArgumentsParser {
         type: 'array',
         requiresArg: true,
       })
-      // TODO(cemmer): don't allow dir2dat & --dat
+      .check((checkArgv) => {
+        if (checkArgv.help) {
+          return true;
+        }
+        if (checkArgv.dat && checkArgv.dat.length > 0 && checkArgv._.includes('dir2dat')) {
+          throw new ExpectedError('Argument "--dat" cannot be used with the command "dir2dat"');
+        }
+        return true;
+      })
       .option('dat-exclude', {
         group: groupDatInput,
         description: 'Path(s) to DAT files or archives to exclude from processing (supports globbing)',
@@ -331,38 +346,38 @@ export default class ArgumentsParser {
       })
 
       .option('output', {
-        group: groupRomOutput,
+        group: groupRomOutputPath,
         alias: 'o',
         description: 'Path to the ROM output directory (supports replaceable symbols, see below)',
         type: 'string',
         coerce: ArgumentsParser.getLastValue, // don't allow string[] values
         requiresArg: true,
       })
       .option('dir-mirror', {
-        group: groupRomOutput,
+        group: groupRomOutputPath,
         description: 'Use the input subdirectory structure for the output directory',
         type: 'boolean',
       })
       .option('dir-dat-name', {
-        group: groupRomOutput,
+        group: groupRomOutputPath,
         alias: 'D',
         description: 'Use the DAT name as the output subdirectory',
         type: 'boolean',
         implies: 'dat',
       })
       .option('dir-dat-description', {
-        group: groupRomOutput,
+        group: groupRomOutputPath,
         description: 'Use the DAT description as the output subdirectory',
         type: 'boolean',
         implies: 'dat',
       })
       .option('dir-letter', {
-        group: groupRomOutput,
+        group: groupRomOutputPath,
         description: 'Group games in an output subdirectory by the first --dir-letter-count letters in their name',
         type: 'boolean',
       })
       .option('dir-letter-count', {
-        group: groupRomOutput,
+        group: groupRomOutputPath,
         description: 'How many game name letters to use for the subdirectory name',
         type: 'number',
         coerce: (val: number) => Math.max(ArgumentsParser.getLastValue(val), 1),
@@ -377,21 +392,21 @@ export default class ArgumentsParser {
         return true;
       })
       .option('dir-letter-limit', {
-        group: groupRomOutput,
+        group: groupRomOutputPath,
         description: 'Limit the number of games in letter subdirectories, splitting into multiple subdirectories if necessary',
         type: 'number',
         coerce: (val: number) => Math.max(ArgumentsParser.getLastValue(val), 1),
         requiresArg: true,
         implies: 'dir-letter',
       })
       .option('dir-letter-group', {
-        group: groupRomOutput,
+        group: groupRomOutputPath,
         description: 'Group letter subdirectories into ranges, combining multiple letters together (requires --dir-letter-limit)',
         type: 'boolean',
         implies: 'dir-letter-limit',
       })
       .option('dir-game-subdir', {
-        group: groupRomOutput,
+        group: groupRomOutputPath,
         description: 'Append the name of the game as an output subdirectory depending on its ROMs',
         choices: Object.keys(GameSubdirMode)
           .filter((mode) => Number.isNaN(Number(mode)))
@@ -400,6 +415,17 @@ export default class ArgumentsParser {
         requiresArg: true,
         default: GameSubdirMode[GameSubdirMode.MULTIPLE].toLowerCase(),
       })
+
+      .option('rom-fix-extension', {
+        group: groupRomOutput,
+        description: `Read ROMs for known file signatures and use the correct extension (also affects dir2dat) (supported: ${ROMSignature.getSupportedExtensions().join(', ')})`,
+        choices: Object.keys(RomFixExtension)
+          .filter((mode) => Number.isNaN(Number(mode)))
+          .map((mode) => mode.toLowerCase()),
+        coerce: ArgumentsParser.getLastValue, // don't allow string[] values
+        requiresArg: true,
+        default: RomFixExtension[RomFixExtension.AUTO].toLowerCase(),
+      })
       .option('overwrite', {
         group: groupRomOutput,
         alias: 'O',