Feature: handle ROMs with headers when calculating checksums (#47)

.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+end_of_line = lf

.gitattributes

@@ -1,2 +1,4 @@
 # Stop `core.autocrlf true`
+*.lnx binary
+*.nes binary
 *.rom binary

.gitignore

@@ -106,7 +106,8 @@ dist
 
 # Custom
 build/
-demo-magic.sh
+*.bat
+*.sh
 
 # DATs
 *.dat

README.md

@@ -32,10 +32,10 @@ With a large ROM collection it can be difficult to:
 
 `igir` needs two sets of files:
 
-1. ROMs, of course!
+1. ROMs, including ones with [headers](https://no-intro.org/faq.htm)
 2. One or more DATs ([see below](#what-are-dats) for where to download)
 
-Many different input archive types are supported: .001, .7z, .bz2, .gz, .rar, .tar, .xz, .z, .z01, .zip, .zipx, and more!
+Many different input archive types are supported for both ROMs and DATs: .001, .7z, .bz2, .gz, .rar, .tar, .tgz, .xz, .z, .z01, .zip, .zipx, and more!
 
 `igir` then needs one or more commands:
 
@@ -57,8 +57,8 @@ npx igir@latest [commands..] [options]
 Here is the full `igir --help` message which shows all available options and a number of common use case examples:
 
 ```help
- ______   ______   ______  _______  
-|      \ /      \ |      \|       \ 
+ ______   ______   ______  _______
+|      \ /      \ |      \|       \
  \$$$$$$|  $$$$$$\ \$$$$$$| $$$$$$$\
   | $$  | $$ __\$$  | $$  | $$__| $$
   | $$  | $$|    \  | $$  | $$    $$
@@ -85,6 +85,9 @@ Path options (inputs support globbing):
   -I, --input-exclude  Path(s) to ROM files to exclude                                  [array]
   -o, --output         Path to the ROM output directory                                [string]
 
+Input options:
+  -H, --header  Glob pattern of files to force header processing for                   [string]
+
 Output options:
       --dir-mirror    Use the input subdirectory structure for output subdirectories  [boolean]
   -D, --dir-dat-name  Use the DAT name as the output subdirectory                     [boolean]
@@ -95,7 +98,7 @@ Output options:
   -Z, --zip-exclude   Glob pattern of files to exclude from zipping                    [string]
   -O, --overwrite     Overwrite any ROMs in the output directory                      [boolean]
 
-Priority options:
+Priority options (requires --single):
       --prefer-good            Prefer good ROM dumps over bad                         [boolean]
   -l, --prefer-language        List of comma-separated languages in priority order (supported:
                                DA, DE, EL, EN, ES, FI, FR, IT, JA, KO, NL, NO, PT, RU, SV, ZH)
@@ -197,11 +200,12 @@ There a few different popular ROM managers that have similar features:
 
 Each manager has its own pros, but most share the same cons:
 
-- Windows-only (sometimes with Wine support), making management on macOS and Linux difficult 
+- Windows-only (sometimes with Wine support), making management on macOS and Linux difficult
 - Limited CLI support, making batching and repeatable actions difficult
 - UIs that don't clearly state what actions can, will, or are being performed
 - Required proprietary database setup step
 - Limited or nonexistent archive extraction support
+- Limited or nonexistent ROM header support
 - Limited or nonexistent parent/clone, region, language, version, and ROM type filtering
 - Limited or nonexistent priorities when creating a 1G1R set
 - Limited or nonexistent folder management options

package.json

@@ -5,6 +5,7 @@
   "keywords": [
     "1g1r",
     "emulation",
+    "logiqx",
     "no-intro",
     "roms"
   ],

scripts/update-readme-help.sh

@@ -9,7 +9,7 @@ cd "$(dirname "$0")/.."
 
 
 README="README.md"
-HELP="$(./node_modules/.bin/ts-node ./index.ts --help "${1:-80}")"
+HELP="$(./node_modules/.bin/ts-node ./index.ts --help "${1:-80}" | sed 's/ *$//g')"
 (awk 'BEGIN {msg=ARGV[1]; delete ARGV[1]; p=1} /^```help/ {print; print msg; p=0} /^```$/ {p=1} p' \
   "${HELP}" \
   "${README}" > "${README}.temp" || exit 1) && mv -f "${README}.temp" "${README}"

src/console/progressBar.ts

@@ -5,6 +5,7 @@ import LogLevel from './logLevel.js';
 export const Symbols: { [key: string]: string } = {
   WAITING: chalk.grey('⋯'),
   SEARCHING: chalk.magenta('↻'),
+  HASHING: chalk.magenta('#'),
   GENERATING: chalk.cyan('Σ'),
   PROCESSING: chalk.cyan('⚙'),
   FILTERING: chalk.cyan('∆'),

src/console/singleBarFormatted.ts

@@ -123,7 +123,7 @@ export default class SingleBarFormatted {
     const seconds = this.eta as number;
     const secondsRounded = 5 * Math.round(seconds / 5);
     if (secondsRounded >= 3600) {
-      return `${Math.floor(secondsRounded / 3600)}h${(secondsRounded % 3600) / 60}m`;
+      return `${Math.floor(secondsRounded / 3600)}h${Math.floor((secondsRounded % 3600) / 60)}m`;
     } if (secondsRounded >= 60) {
       return `${Math.floor(secondsRounded / 60)}m${(secondsRounded % 60)}s`;
     } if (seconds >= 10) {

src/constants.ts

@@ -17,7 +17,14 @@ export default class Constants {
 
   static readonly ROM_SCANNER_THREADS = 25;
 
+  // TODO(cemmer): is there a way to set a global limit with only one DAT? semaphores?
+  static readonly ROM_HEADER_HASHER_THREADS = Math.ceil(
+    Constants.ROM_SCANNER_THREADS / Constants.DAT_THREADS,
+  );
+
   static readonly ROM_WRITER_THREADS = Math.ceil(
     Constants.ROM_SCANNER_THREADS / Constants.DAT_THREADS,
   );
+
+  static readonly FILE_READING_CHUNK_SIZE = 1024 * 1024; // 1MiB
 }

src/igir.ts

@@ -7,6 +7,7 @@ import Constants from './constants.js';
 import CandidateFilter from './modules/candidateFilter.js';
 import CandidateGenerator from './modules/candidateGenerator.js';
 import DATScanner from './modules/datScanner.js';
+import HeaderProcessor from './modules/headerProcessor.js';
 import OutputCleaner from './modules/outputCleaner.js';
 import ReportGenerator from './modules/reportGenerator.js';
 import ROMScanner from './modules/romScanner.js';
@@ -29,13 +30,17 @@ export default class Igir {
   }
 
   async main(): Promise<void> {
+    // Scan and process input files
     const dats = await this.processDATScanner();
-    const romFiles = await this.processROMScanner();
+    const rawRomFiles = await this.processROMScanner();
+    const processedRomFiles = await this.processHeaderProcessor(rawRomFiles);
 
+    // Set up progress bar and input for DAT processing
     const datProcessProgressBar = this.logger.addProgressBar('Processing DATs', Symbols.PROCESSING, dats.length);
     const datsToWrittenRoms = new Map<DAT, Map<Parent, File[]>>();
     const datsStatuses: DATStatus[] = [];
 
+    // Process every DAT
     await async.eachLimit(dats, Constants.DAT_THREADS, async (dat, callback) => {
       const progressBar = this.logger.addProgressBar(
         dat.getNameShort(),
@@ -45,7 +50,8 @@ export default class Igir {
       await datProcessProgressBar.increment();
 
       // Generate and filter ROM candidates
-      const romCandidates = await new CandidateGenerator(progressBar).generate(dat, romFiles);
+      const romCandidates = await new CandidateGenerator(progressBar)
+        .generate(dat, processedRomFiles);
       const romOutputs = await new CandidateFilter(this.options, progressBar)
         .filter(dat, romCandidates);
 
@@ -98,6 +104,14 @@ export default class Igir {
     return romInputs;
   }
 
+  private async processHeaderProcessor(romFiles: File[]): Promise<File[]> {
+    const headerProcessorProgressBar = this.logger.addProgressBar('Reading ROM headers', Symbols.WAITING);
+    const processedRomFiles = await new HeaderProcessor(this.options, headerProcessorProgressBar)
+      .process(romFiles);
+    await headerProcessorProgressBar.doneItems(processedRomFiles.length, 'file', 'read');
+    return processedRomFiles;
+  }
+
   private async processOutputCleaner(
     datsToWrittenRoms: Map<DAT, Map<Parent, File[]>>,
   ): Promise<void> {

src/modules/argumentsParser.ts

@@ -43,10 +43,11 @@ export default class ArgumentsParser {
     this.logger.info(`Parsing CLI arguments: ${argv}`);
 
     const groupInputOutputPaths = 'Path options (inputs support globbing):';
+    const groupInput = 'Input options:';
     const groupOutput = 'Output options:';
-    const groupPriority = 'Priority options:';
+    const groupPriority = 'Priority options (requires --single):';
     const groupFiltering = 'Filtering options:';
-    const groupDebug = 'Debug options:';
+    const groupHelp = 'Help options:';
 
     // Add every command to a yargs object, recursively, resulting in the ability to specify
     // multiple commands
@@ -128,6 +129,15 @@ export default class ArgumentsParser {
         return true;
       })
 
+      .option('header', {
+        group: groupInput,
+        alias: 'H',
+        description: 'Glob pattern of files to force header processing for',
+        type: 'string',
+        coerce: ArgumentsParser.getLastValue, // don't allow string[] values
+        requiresArg: true,
+      })
+
       .option('dir-mirror', {
         group: groupOutput,
         description: 'Use the input subdirectory structure for output subdirectories',
@@ -297,7 +307,7 @@ export default class ArgumentsParser {
       })
 
       .option('verbose', {
-        group: groupDebug,
+        group: groupHelp,
         alias: 'v',
         description: 'Enable verbose logging, can specify twice (-vv)',
         type: 'count',
@@ -317,8 +327,9 @@ export default class ArgumentsParser {
         ['$0 copy -i ROMs/ -o /media/SDCard/ROMs/ -D --dir-letter -t', 'Copy ROMs to a flash cart and test them'],
       ])
 
-    // Colorize help output
+      // Colorize help output
       .option('help', {
+        group: groupHelp,
         alias: 'h',
         description: 'Show help',
         type: 'boolean',

src/modules/candidateGenerator.ts

@@ -31,13 +31,14 @@ export default class CandidateGenerator {
       return output;
     }
 
+    await this.progressBar.setSymbol(Symbols.GENERATING);
+    await this.progressBar.reset(dat.getParents().length);
+
+    // TODO(cemmer): use filesize combined with CRC for indexing
     // TODO(cemmer): ability to index files by some other property such as name
     const crc32ToInputFiles = await CandidateGenerator.indexFilesByCrc(inputRomFiles);
     await this.progressBar.logInfo(`${dat.getName()}: ${crc32ToInputFiles.size} unique ROM CRC32s found`);
 
-    await this.progressBar.setSymbol(Symbols.GENERATING);
-    await this.progressBar.reset(dat.getParents().length);
-
     // TODO(cemmer): ability to work without DATs, generating a parent/game/release per file
     // For each parent, try to generate a parent candidate
     /* eslint-disable no-await-in-loop */
@@ -79,20 +80,25 @@ export default class CandidateGenerator {
   private static async indexFilesByCrc(files: File[]): Promise<Map<string, File>> {
     return files.reduce(async (accPromise, file) => {
       const acc = await accPromise;
-      if (acc.has(await file.getCrc32())) {
-        // Have already seen file, prefer non-archived files
-        const existing = acc.get(await file.getCrc32()) as File;
-        if (!(file instanceof ArchiveEntry) && existing instanceof ArchiveEntry) {
-          acc.set(await file.getCrc32(), file);
-        }
-      } else {
-        // Haven't seen file yet, store it
-        acc.set(await file.getCrc32(), file);
-      }
+      this.addToIndex(acc, await file.getCrc32(), file);
+      this.addToIndex(acc, await file.getCrc32WithoutHeader(), file);
       return acc;
     }, Promise.resolve(new Map<string, File>()));
   }
 
+  private static addToIndex(map: Map<string, File>, hash: string, file: File): void {
+    if (map.has(hash)) {
+      // Have already seen file, prefer non-archived files
+      const existing = map.get(hash) as File;
+      if (!(file instanceof ArchiveEntry) && existing instanceof ArchiveEntry) {
+        map.set(hash, file);
+      }
+    } else {
+      // Haven't seen file yet, store it
+      map.set(hash, file);
+    }
+  }
+
   private async buildReleaseCandidateForRelease(
     game: Game,
     release: Release | undefined,

src/modules/headerProcessor.ts

@@ -0,0 +1,60 @@
+import async, { AsyncResultCallback } from 'async';
+
+import ProgressBar, { Symbols } from '../console/progressBar.js';
+import Constants from '../constants.js';
+import File from '../types/files/file.js';
+import FileHeader from '../types/files/fileHeader.js';
+import Options from '../types/options.js';
+
+// TODO(cemmer): put debug statements in here
+export default class HeaderProcessor {
+  private readonly options: Options;
+
+  private readonly progressBar: ProgressBar;
+
+  constructor(options: Options, progressBar: ProgressBar) {
+    this.options = options;
+    this.progressBar = progressBar;
+  }
+
+  async process(inputRomFiles: File[]): Promise<File[]> {
+    await this.progressBar.logInfo('Processing file headers');
+
+    await this.progressBar.setSymbol(Symbols.HASHING);
+    await this.progressBar.reset(inputRomFiles.length);
+
+    return async.mapLimit(
+      inputRomFiles,
+      Constants.ROM_HEADER_HASHER_THREADS,
+      async (inputFile, callback: AsyncResultCallback<File, Error>) => {
+        await this.progressBar.increment();
+
+        // Don't do anything if the file already has a header
+        if (inputFile.getFileHeader()) {
+          return callback(null, inputFile);
+        }
+
+        // Can get FileHeader from extension, use that
+        const headerForExtension = FileHeader.getForFilename(inputFile.getExtractedFilePath());
+        if (headerForExtension) {
+          const fileWithHeader = await inputFile.withFileHeader(headerForExtension).resolve();
+          return callback(null, fileWithHeader);
+        }
+
+        // Should get FileHeader from File, try to
+        if (this.options.shouldReadFileForHeader(inputFile.getExtractedFilePath())) {
+          const headerForFile = await inputFile
+            .extract(async (localFile) => FileHeader.getForFileContents(localFile));
+          if (headerForFile) {
+            const fileWithHeader = await inputFile.withFileHeader(headerForFile).resolve();
+            return callback(null, fileWithHeader);
+          }
+          await this.progressBar.logWarn(`Couldn't detect header for ${inputFile.toString()}`);
+        }
+
+        // Should not get FileHeader
+        return callback(null, inputFile);
+      },
+    );
+  }
+}

src/modules/romScanner.ts

@@ -12,7 +12,6 @@ import Scanner from './scanner.js';
  * This class will not be run concurrently with any other class.
  */
 export default class ROMScanner extends Scanner {
-  // TODO(cemmer): support for headered ROM files (e.g. NES)
   async scan(): Promise<File[]> {
     await this.progressBar.setSymbol(Symbols.SEARCHING);
     await this.progressBar.reset(0);

src/modules/romWriter.ts

@@ -6,9 +6,9 @@ import path from 'path';
 import ProgressBar, { Symbols } from '../console/progressBar.js';
 import Constants from '../constants.js';
 import fsPoly from '../polyfill/fsPoly.js';
+import Zip from '../types/archives/zip.js';
 import ArchiveEntry from '../types/files/archiveEntry.js';
 import File from '../types/files/file.js';
-import Zip from '../types/files/zip.js';
 import DAT from '../types/logiqx/dat.js';
 import Parent from '../types/logiqx/parent.js';
 import ROM from '../types/logiqx/rom.js';
@@ -107,8 +107,10 @@ export default class ROMWriter {
     const crcToRoms = releaseCandidate.getRomsByCrc32();
 
     return releaseCandidate.getFiles().reduce(async (accPromise, inputFile) => {
+      // TODO(cemmer): use filesize combined with CRC for indexing
       const acc = await accPromise;
-      const rom = crcToRoms.get(await inputFile.getCrc32()) as ROM;
+      const rom = crcToRoms.get(await inputFile.getCrc32())
+               || crcToRoms.get(await inputFile.getCrc32WithoutHeader()) as ROM;
 
       let outputFile: File;
       if (this.options.shouldZip(rom.getName())) {
@@ -221,9 +223,11 @@ export default class ROMWriter {
 
     // If the file in the output zip already exists and has the same CRC then do nothing
     const existingOutputEntry = outputZip.getEntry(outputRomFile.getEntryPath() as string);
-    if (existingOutputEntry?.header.crc === parseInt(await outputRomFile.getCrc32(), 16)) {
-      await this.progressBar.logDebug(`${outputZipPath}: ${outputRomFile.getEntryPath()} already exists`);
-      return false;
+    if (existingOutputEntry) {
+      if (existingOutputEntry.header.crc === parseInt(await outputRomFile.getCrc32(), 16)) {
+        await this.progressBar.logDebug(`${outputZipPath}: ${outputRomFile.getEntryPath()} already exists`);
+        return false;
+      }
     }
 
     // Write the entry