Refactor: consolidate file signature code (#1225)

docs/output/options.md

@@ -22,15 +22,15 @@ Here are some examples of common mistakes:
 
 This correction behavior can be controlled with the following option:
 
-- `--rom-fix-extension never`
+- `--fix-extension never`
 
   Don't correct any ROM filename extensions. If a DAT doesn't provide a ROM filename, a default name of `<game name>.rom` will be used.
 
-- `--rom-fix-extension auto` (default)
+- `--fix-extension auto` (default)
 
   When not using DATs (no [`--dat <path>` option](../dats/processing.md) was provided), or when a DAT doesn't specify the filename for a ROM, then try to correct the filename extension.
 
-- `--rom-fix-extension always`
+- `--fix-extension always`
 
   Always try to correct filename extensions, ignoring the information provided by DATs. You likely don't want this option.
 

src/modules/argumentsParser.ts

@@ -10,13 +10,12 @@ import ConsolePoly from '../polyfill/consolePoly.js';
 import ExpectedError from '../types/expectedError.js';
 import { ChecksumBitmask } from '../types/files/fileChecksums.js';
 import ROMHeader from '../types/files/romHeader.js';
-import ROMSignature from '../types/files/romSignature.js';
 import Internationalization from '../types/internationalization.js';
 import Options, {
+  FixExtension,
   GameSubdirMode,
   InputChecksumArchivesMode,
   MergeMode,
-  RomFixExtension,
 } from '../types/options.js';
 import PatchFactory from '../types/patches/patchFactory.js';
 
@@ -416,15 +415,15 @@ export default class ArgumentsParser {
         default: GameSubdirMode[GameSubdirMode.MULTIPLE].toLowerCase(),
       })
 
-      .option('rom-fix-extension', {
+      .option('fix-extension', {
         group: groupRomOutput,
-        description: `Read ROMs for known file signatures and use the correct extension (also affects dir2dat) (supported: ${ROMSignature.getSupportedExtensions().join(', ')})`,
-        choices: Object.keys(RomFixExtension)
+        description: 'Read files for known signatures and use the correct extension (also affects dir2dat)',
+        choices: Object.keys(FixExtension)
           .filter((mode) => Number.isNaN(Number(mode)))
           .map((mode) => mode.toLowerCase()),
         coerce: ArgumentsParser.getLastValue, // don't allow string[] values
         requiresArg: true,
-        default: RomFixExtension[RomFixExtension.AUTO].toLowerCase(),
+        default: FixExtension[FixExtension.AUTO].toLowerCase(),
       })
       .option('overwrite', {
         group: groupRomOutput,

src/modules/candidateExtensionCorrector.ts

@@ -7,8 +7,8 @@ import DAT from '../types/dats/dat.js';
 import Parent from '../types/dats/parent.js';
 import ROM from '../types/dats/rom.js';
 import ArchiveEntry from '../types/files/archives/archiveEntry.js';
-import ROMSignature from '../types/files/romSignature.js';
-import Options, { RomFixExtension } from '../types/options.js';
+import FileSignature from '../types/files/fileSignature.js';
+import Options, { FixExtension } from '../types/options.js';
 import OutputFactory from '../types/outputFactory.js';
 import ReleaseCandidate from '../types/releaseCandidate.js';
 import ROMWithFiles from '../types/romWithFiles.js';
@@ -62,8 +62,8 @@ export default class CandidateExtensionCorrector extends Module {
   }
 
   private romNeedsCorrecting(romWithFiles: ROMWithFiles): boolean {
-    return this.options.getRomFixExtension() === RomFixExtension.ALWAYS
-      || (this.options.getRomFixExtension() === RomFixExtension.AUTO && (
+    return this.options.getFixExtension() === FixExtension.ALWAYS
+      || (this.options.getFixExtension() === FixExtension.AUTO && (
         !this.options.usingDats()
           || romWithFiles.getRom().getName().trim() === ''
       ));
@@ -149,7 +149,7 @@ export default class CandidateExtensionCorrector extends Module {
         .toString()}`);
 
       await romWithFiles.getInputFile().createReadStream(async (stream) => {
-        const romSignature = await ROMSignature.signatureFromFileStream(stream);
+        const romSignature = await FileSignature.signatureFromFileStream(stream);
         if (!romSignature) {
           // No signature was found, so we can't perform any correction
           return;

src/types/files/archives/archive.ts

@@ -21,7 +21,7 @@ export default abstract class Archive {
     return this.filePath;
   }
 
-  abstract getArchiveEntries(checksumBitmask: number): Promise<ArchiveEntry<this>[]>;
+  abstract getArchiveEntries(checksumBitmask: number): Promise<ArchiveEntry<Archive>[]>;
 
   abstract extractEntryToFile(
     entryPath: string,

src/types/files/archives/gzip.ts

@@ -1,4 +1,7 @@
+import Archive from './archive.js';
+import ArchiveEntry from './archiveEntry.js';
 import SevenZip from './sevenZip.js';
+import Tar from './tar.js';
 
 export default class Gzip extends SevenZip {
   // eslint-disable-next-line class-methods-use-this
@@ -15,9 +18,12 @@ export default class Gzip extends SevenZip {
     return Gzip.getExtensions()[0];
   }
 
-  static getFileSignatures(): Buffer[] {
-    return [
-      Buffer.from('1F8B08', 'hex'), // deflate
-    ];
+  async getArchiveEntries(checksumBitmask: number): Promise<ArchiveEntry<Archive>[]> {
+    // See if this file is actually a .tar.gz
+    try {
+      return await new Tar(this.getFilePath()).getArchiveEntries(checksumBitmask);
+    } catch { /* empty */ }
+
+    return super.getArchiveEntries(checksumBitmask);
   }
 }

src/types/files/archives/rar.ts

@@ -26,13 +26,6 @@ export default class Rar extends Archive {
     return Rar.getExtensions()[0];
   }
 
-  static getFileSignatures(): Buffer[] {
-    return [
-      Buffer.from('526172211A0700', 'hex'), // v1.50+
-      Buffer.from('526172211A070100', 'hex'), // v5.00+
-    ];
-  }
-
   async getArchiveEntries(checksumBitmask: number): Promise<ArchiveEntry<this>[]> {
     const rar = await unrar.createExtractorFromFile({
       filepath: this.getFilePath(),

src/types/files/archives/sevenZip.ts

@@ -28,11 +28,7 @@ export default class SevenZip extends Archive {
     return SevenZip.getExtensions()[0];
   }
 
-  static getFileSignatures(): Buffer[] {
-    return [Buffer.from('377ABCAF271C', 'hex')];
-  }
-
-  async getArchiveEntries(checksumBitmask: number): Promise<ArchiveEntry<this>[]> {
+  async getArchiveEntries(checksumBitmask: number): Promise<ArchiveEntry<Archive>[]> {
     /**
      * WARN(cemmer): even with the above mutex, {@link _7z.list} will still sometimes return no
      *  entries. Most archives contain at least one file, so assume this is wrong and attempt

src/types/files/archives/tar.ts

@@ -29,16 +29,6 @@ export default class Tar extends Archive {
     return path.parse(this.getFilePath()).ext;
   }
 
-  static getFileSignatures(): Buffer[] {
-    return [
-      // .tar
-      Buffer.from('7573746172003030', 'hex'),
-      Buffer.from('7573746172202000', 'hex'),
-      // .tar.gz / .tgz
-      Buffer.from('1F8B08', 'hex'), // deflate
-    ];
-  }
-
   async getArchiveEntries(checksumBitmask: number): Promise<ArchiveEntry<this>[]> {
     const archiveEntryPromises: Promise<ArchiveEntry<this>>[] = [];
 

src/types/files/archives/z.ts

@@ -14,11 +14,4 @@ export default class Z extends SevenZip {
   getExtension(): string {
     return Z.getExtensions()[0];
   }
-
-  static getFileSignatures(): Buffer[] {
-    return [
-      Buffer.from('1F9D', 'hex'), // LZW compression
-      Buffer.from('1FA0', 'hex'), // LZH compression
-    ];
-  }
 }

src/types/files/archives/zip.ts

@@ -31,13 +31,6 @@ export default class Zip extends Archive {
     return Zip.getExtensions()[0];
   }
 
-  static getFileSignatures(): Buffer[] {
-    return [
-      Buffer.from('504B0304', 'hex'),
-      Buffer.from('504B0506', 'hex'), // empty archive
-    ];
-  }
-
   async getArchiveEntries(checksumBitmask: number): Promise<ArchiveEntry<this>[]> {
     // https://github.com/ZJONSSON/node-unzipper/issues/280
     // UTF-8 entry names are not decoded correctly

src/types/files/archives/zipSpanned.ts

@@ -14,8 +14,4 @@ export default class ZipSpanned extends SevenZip {
   getExtension(): string {
     return ZipSpanned.getExtensions()[0];
   }
-
-  static getFileSignatures(): Buffer[] {
-    return [Buffer.from('504B0708', 'hex')];
-  }
 }

src/types/files/archives/zipX.ts

@@ -21,8 +21,4 @@ export default class ZipX extends SevenZip {
     }
     return path.parse(this.getFilePath()).ext;
   }
-
-  static getFileSignatures(): Buffer[] {
-    return [];
-  }
 }

src/types/files/fileCache.ts

@@ -133,7 +133,7 @@ export default class FileCache {
   static async getOrComputeEntries<T extends Archive>(
     archive: T,
     checksumBitmask: number,
-  ): Promise<ArchiveEntry<T>[]> {
+  ): Promise<ArchiveEntry<Archive>[]> {
     if (!this.enabled || checksumBitmask === ChecksumBitmask.NONE) {
       return archive.getArchiveEntries(checksumBitmask);
     }

src/types/files/fileFactory.ts

@@ -1,6 +1,3 @@
-import fs from 'node:fs';
-import path from 'node:path';
-
 import ExpectedError from '../expectedError.js';
 import Archive from './archives/archive.js';
 import ArchiveEntry from './archives/archiveEntry.js';
@@ -16,6 +13,7 @@ import ZipX from './archives/zipX.js';
 import File from './file.js';
 import FileCache from './fileCache.js';
 import { ChecksumBitmask } from './fileChecksums.js';
+import FileSignature from './fileSignature.js';
 
 export default class FileFactory {
   static async filesFrom(
@@ -31,7 +29,11 @@ export default class FileFactory {
     }
 
     try {
-      return await this.entriesFromArchiveExtension(filePath, checksumBitmask);
+      const entries = await this.entriesFromArchiveExtension(filePath, checksumBitmask);
+      if (entries !== undefined) {
+        return entries;
+      }
+      return [await this.fileFrom(filePath, checksumBitmask)];
     } catch (error) {
       if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
         throw new ExpectedError(`file doesn't exist: ${filePath}`);
@@ -69,26 +71,27 @@ export default class FileFactory {
   private static async entriesFromArchiveExtension(
     filePath: string,
     checksumBitmask: number,
-  ): Promise<ArchiveEntry<Archive>[]> {
+    fileExt = filePath.replace(/.+?(?=(\.[a-zA-Z0-9]+)+)/, ''),
+  ): Promise<ArchiveEntry<Archive>[] | undefined> {
     let archive: Archive;
-    if (Zip.getExtensions().some((ext) => filePath.toLowerCase().endsWith(ext))) {
+    if (Zip.getExtensions().some((ext) => fileExt.toLowerCase().endsWith(ext))) {
       archive = new Zip(filePath);
-    } else if (Tar.getExtensions().some((ext) => filePath.toLowerCase().endsWith(ext))) {
+    } else if (Tar.getExtensions().some((ext) => fileExt.toLowerCase().endsWith(ext))) {
       archive = new Tar(filePath);
-    } else if (Rar.getExtensions().some((ext) => filePath.toLowerCase().endsWith(ext))) {
+    } else if (Rar.getExtensions().some((ext) => fileExt.toLowerCase().endsWith(ext))) {
       archive = new Rar(filePath);
-    } else if (Gzip.getExtensions().some((ext) => filePath.toLowerCase().endsWith(ext))) {
+    } else if (Gzip.getExtensions().some((ext) => fileExt.toLowerCase().endsWith(ext))) {
       archive = new Gzip(filePath);
-    } else if (SevenZip.getExtensions().some((ext) => filePath.toLowerCase().endsWith(ext))) {
+    } else if (SevenZip.getExtensions().some((ext) => fileExt.toLowerCase().endsWith(ext))) {
       archive = new SevenZip(filePath);
-    } else if (Z.getExtensions().some((ext) => filePath.toLowerCase().endsWith(ext))) {
+    } else if (Z.getExtensions().some((ext) => fileExt.toLowerCase().endsWith(ext))) {
       archive = new Z(filePath);
-    } else if (ZipSpanned.getExtensions().some((ext) => filePath.toLowerCase().endsWith(ext))) {
+    } else if (ZipSpanned.getExtensions().some((ext) => fileExt.toLowerCase().endsWith(ext))) {
       archive = new ZipSpanned(filePath);
-    } else if (ZipX.getExtensions().some((ext) => filePath.toLowerCase().endsWith(ext))) {
+    } else if (ZipX.getExtensions().some((ext) => fileExt.toLowerCase().endsWith(ext))) {
       archive = new ZipX(filePath);
     } else {
-      throw new ExpectedError(`unknown archive type: ${path.extname(filePath)}`);
+      return undefined;
     }
 
     return FileCache.getOrComputeEntries(archive, checksumBitmask);
@@ -104,71 +107,25 @@ export default class FileFactory {
     filePath: string,
     checksumBitmask: number,
   ): Promise<ArchiveEntry<Archive>[] | undefined> {
-    const maxSignatureLengthBytes = [
-      ...Zip.getFileSignatures(),
-      ...Tar.getFileSignatures(),
-      ...Rar.getFileSignatures(),
-      // 7zip
-      ...Gzip.getFileSignatures(),
-      ...SevenZip.getFileSignatures(),
-      ...Z.getFileSignatures(),
-      ...ZipSpanned.getFileSignatures(),
-      ...ZipX.getFileSignatures(),
-    ].reduce((max, signature) => Math.max(max, signature.length), 0);
-
-    let fileSignature: Buffer;
+    let signature: FileSignature | undefined;
     try {
-      const stream = fs.createReadStream(filePath, { end: maxSignatureLengthBytes });
-      fileSignature = await new Promise<Buffer>((resolve, reject) => {
-        const chunks: Buffer[] = [];
-        stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
-        stream.on('end', () => resolve(Buffer.concat(chunks)));
-        stream.on('error', reject);
-      });
-      stream.destroy();
+      const file = await File.fileOf({ filePath });
+      signature = await file.createReadStream(
+        async (stream) => FileSignature.signatureFromFileStream(stream),
+      );
     } catch {
       // Fail silently on assumed I/O errors
       return undefined;
     }
-
-    let archive: Archive;
-    if (Zip.getFileSignatures()
-      .some((sig) => fileSignature.subarray(0, sig.length).equals(sig))
-    ) {
-      archive = new Zip(filePath);
-    } else if (Tar.getFileSignatures()
-      .some((sig) => fileSignature.subarray(0, sig.length).equals(sig))
-    ) {
-      archive = new Tar(filePath);
-    } else if (Rar.getFileSignatures()
-      .some((sig) => fileSignature.subarray(0, sig.length).equals(sig))
-    ) {
-      archive = new Rar(filePath);
-    } else if (Gzip.getFileSignatures()
-      .some((sig) => fileSignature.subarray(0, sig.length).equals(sig))
-    ) {
-      archive = new Gzip(filePath);
-    } else if (SevenZip.getFileSignatures()
-      .some((sig) => fileSignature.subarray(0, sig.length).equals(sig))
-    ) {
-      archive = new SevenZip(filePath);
-    } else if (Z.getFileSignatures()
-      .some((sig) => fileSignature.subarray(0, sig.length).equals(sig))
-    ) {
-      archive = new Z(filePath);
-    } else if (ZipSpanned.getFileSignatures()
-      .some((sig) => fileSignature.subarray(0, sig.length).equals(sig))
-    ) {
-      archive = new ZipSpanned(filePath);
-    } else if (ZipX.getFileSignatures()
-      .some((sig) => fileSignature.subarray(0, sig.length).equals(sig))
-    ) {
-      archive = new ZipX(filePath);
-    } else {
+    if (!signature) {
       return undefined;
     }
 
-    return FileCache.getOrComputeEntries(archive, checksumBitmask);
+    return this.entriesFromArchiveExtension(
+      filePath,
+      checksumBitmask,
+      signature.getExtension(),
+    );
   }
 
   static isExtensionArchive(filePath: string): boolean {