diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..74fb9e0be --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*] +indent_style = space +indent_size = 2 +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true +end_of_line = lf diff --git a/.gitattributes b/.gitattributes index 6727e0cf4..09c000ee3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,4 @@ # Stop `core.autocrlf true` +*.lnx binary +*.nes binary *.rom binary diff --git a/.gitignore b/.gitignore index 2882c5760..2173f090e 100644 --- a/.gitignore +++ b/.gitignore @@ -106,7 +106,8 @@ dist # Custom build/ -demo-magic.sh +*.bat +*.sh # DATs *.dat diff --git a/README.md b/README.md index d0073eddb..18ea399a0 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,10 @@ With a large ROM collection it can be difficult to: `igir` needs two sets of files: -1. ROMs, of course! +1. ROMs, including ones with [headers](https://no-intro.org/faq.htm) 2. One or more DATs ([see below](#what-are-dats) for where to download) -Many different input archive types are supported: .001, .7z, .bz2, .gz, .rar, .tar, .xz, .z, .z01, .zip, .zipx, and more! +Many different input archive types are supported for both ROMs and DATs: .001, .7z, .bz2, .gz, .rar, .tar, .tgz, .xz, .z, .z01, .zip, .zipx, and more! `igir` then needs one or more commands: @@ -57,8 +57,8 @@ npx igir@latest [commands..] [options] Here is the full `igir --help` message which shows all available options and a number of common use case examples: ```help - ______ ______ ______ _______ -| \ / \ | \| \ + ______ ______ ______ _______ +| \ / \ | \| \ \$$$$$$| $$$$$$\ \$$$$$$| $$$$$$$\ | $$ | $$ __\$$ | $$ | $$__| $$ | $$ | $$| \ | $$ | $$ $$ @@ -85,6 +85,9 @@ Path options (inputs support globbing): -I, --input-exclude Path(s) to ROM files to exclude [array] -o, --output Path to the ROM output directory [string] +Input options: + -H, --header Glob pattern of files to force header processing for [string] + Output options: --dir-mirror Use the input subdirectory structure for output subdirectories [boolean] -D, --dir-dat-name Use the DAT name as the output subdirectory [boolean] @@ -95,7 +98,7 @@ Output options: -Z, --zip-exclude Glob pattern of files to exclude from zipping [string] -O, --overwrite Overwrite any ROMs in the output directory [boolean] -Priority options: +Priority options (requires --single): --prefer-good Prefer good ROM dumps over bad [boolean] -l, --prefer-language List of comma-separated languages in priority order (supported: DA, DE, EL, EN, ES, FI, FR, IT, JA, KO, NL, NO, PT, RU, SV, ZH) @@ -197,11 +200,12 @@ There a few different popular ROM managers that have similar features: Each manager has its own pros, but most share the same cons: -- Windows-only (sometimes with Wine support), making management on macOS and Linux difficult +- Windows-only (sometimes with Wine support), making management on macOS and Linux difficult - Limited CLI support, making batching and repeatable actions difficult - UIs that don't clearly state what actions can, will, or are being performed - Required proprietary database setup step - Limited or nonexistent archive extraction support +- Limited or nonexistent ROM header support - Limited or nonexistent parent/clone, region, language, version, and ROM type filtering - Limited or nonexistent priorities when creating a 1G1R set - Limited or nonexistent folder management options diff --git a/package.json b/package.json index 3bcd0f1b0..04e587688 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "keywords": [ "1g1r", "emulation", + "logiqx", "no-intro", "roms" ], diff --git a/scripts/update-readme-help.sh b/scripts/update-readme-help.sh index 1f7393ce9..cd298c572 100755 --- a/scripts/update-readme-help.sh +++ b/scripts/update-readme-help.sh @@ -9,7 +9,7 @@ cd "$(dirname "$0")/.." README="README.md" -HELP="$(./node_modules/.bin/ts-node ./index.ts --help "${1:-80}")" +HELP="$(./node_modules/.bin/ts-node ./index.ts --help "${1:-80}" | sed 's/ *$//g')" (awk 'BEGIN {msg=ARGV[1]; delete ARGV[1]; p=1} /^```help/ {print; print msg; p=0} /^```$/ {p=1} p' \ "${HELP}" \ "${README}" > "${README}.temp" || exit 1) && mv -f "${README}.temp" "${README}" diff --git a/src/console/progressBar.ts b/src/console/progressBar.ts index e76c4323f..c5f4cdf52 100644 --- a/src/console/progressBar.ts +++ b/src/console/progressBar.ts @@ -5,6 +5,7 @@ import LogLevel from './logLevel.js'; export const Symbols: { [key: string]: string } = { WAITING: chalk.grey('⋯'), SEARCHING: chalk.magenta('↻'), + HASHING: chalk.magenta('#'), GENERATING: chalk.cyan('Σ'), PROCESSING: chalk.cyan('⚙'), FILTERING: chalk.cyan('∆'), diff --git a/src/console/singleBarFormatted.ts b/src/console/singleBarFormatted.ts index a61ec7e69..3d921e60d 100644 --- a/src/console/singleBarFormatted.ts +++ b/src/console/singleBarFormatted.ts @@ -123,7 +123,7 @@ export default class SingleBarFormatted { const seconds = this.eta as number; const secondsRounded = 5 * Math.round(seconds / 5); if (secondsRounded >= 3600) { - return `${Math.floor(secondsRounded / 3600)}h${(secondsRounded % 3600) / 60}m`; + return `${Math.floor(secondsRounded / 3600)}h${Math.floor((secondsRounded % 3600) / 60)}m`; } if (secondsRounded >= 60) { return `${Math.floor(secondsRounded / 60)}m${(secondsRounded % 60)}s`; } if (seconds >= 10) { diff --git a/src/constants.ts b/src/constants.ts index fcd5c79ea..f8f91c77a 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -17,7 +17,14 @@ export default class Constants { static readonly ROM_SCANNER_THREADS = 25; + // TODO(cemmer): is there a way to set a global limit with only one DAT? semaphores? + static readonly ROM_HEADER_HASHER_THREADS = Math.ceil( + Constants.ROM_SCANNER_THREADS / Constants.DAT_THREADS, + ); + static readonly ROM_WRITER_THREADS = Math.ceil( Constants.ROM_SCANNER_THREADS / Constants.DAT_THREADS, ); + + static readonly FILE_READING_CHUNK_SIZE = 1024 * 1024; // 1MiB } diff --git a/src/igir.ts b/src/igir.ts index 4e19d3216..c6daa42d4 100644 --- a/src/igir.ts +++ b/src/igir.ts @@ -7,6 +7,7 @@ import Constants from './constants.js'; import CandidateFilter from './modules/candidateFilter.js'; import CandidateGenerator from './modules/candidateGenerator.js'; import DATScanner from './modules/datScanner.js'; +import HeaderProcessor from './modules/headerProcessor.js'; import OutputCleaner from './modules/outputCleaner.js'; import ReportGenerator from './modules/reportGenerator.js'; import ROMScanner from './modules/romScanner.js'; @@ -29,13 +30,17 @@ export default class Igir { } async main(): Promise { + // Scan and process input files const dats = await this.processDATScanner(); - const romFiles = await this.processROMScanner(); + const rawRomFiles = await this.processROMScanner(); + const processedRomFiles = await this.processHeaderProcessor(rawRomFiles); + // Set up progress bar and input for DAT processing const datProcessProgressBar = this.logger.addProgressBar('Processing DATs', Symbols.PROCESSING, dats.length); const datsToWrittenRoms = new Map>(); const datsStatuses: DATStatus[] = []; + // Process every DAT await async.eachLimit(dats, Constants.DAT_THREADS, async (dat, callback) => { const progressBar = this.logger.addProgressBar( dat.getNameShort(), @@ -45,7 +50,8 @@ export default class Igir { await datProcessProgressBar.increment(); // Generate and filter ROM candidates - const romCandidates = await new CandidateGenerator(progressBar).generate(dat, romFiles); + const romCandidates = await new CandidateGenerator(progressBar) + .generate(dat, processedRomFiles); const romOutputs = await new CandidateFilter(this.options, progressBar) .filter(dat, romCandidates); @@ -98,6 +104,14 @@ export default class Igir { return romInputs; } + private async processHeaderProcessor(romFiles: File[]): Promise { + const headerProcessorProgressBar = this.logger.addProgressBar('Reading ROM headers', Symbols.WAITING); + const processedRomFiles = await new HeaderProcessor(this.options, headerProcessorProgressBar) + .process(romFiles); + await headerProcessorProgressBar.doneItems(processedRomFiles.length, 'file', 'read'); + return processedRomFiles; + } + private async processOutputCleaner( datsToWrittenRoms: Map>, ): Promise { diff --git a/src/modules/argumentsParser.ts b/src/modules/argumentsParser.ts index 23d1e62ed..0c7e66586 100644 --- a/src/modules/argumentsParser.ts +++ b/src/modules/argumentsParser.ts @@ -43,10 +43,11 @@ export default class ArgumentsParser { this.logger.info(`Parsing CLI arguments: ${argv}`); const groupInputOutputPaths = 'Path options (inputs support globbing):'; + const groupInput = 'Input options:'; const groupOutput = 'Output options:'; - const groupPriority = 'Priority options:'; + const groupPriority = 'Priority options (requires --single):'; const groupFiltering = 'Filtering options:'; - const groupDebug = 'Debug options:'; + const groupHelp = 'Help options:'; // Add every command to a yargs object, recursively, resulting in the ability to specify // multiple commands @@ -128,6 +129,15 @@ export default class ArgumentsParser { return true; }) + .option('header', { + group: groupInput, + alias: 'H', + description: 'Glob pattern of files to force header processing for', + type: 'string', + coerce: ArgumentsParser.getLastValue, // don't allow string[] values + requiresArg: true, + }) + .option('dir-mirror', { group: groupOutput, description: 'Use the input subdirectory structure for output subdirectories', @@ -297,7 +307,7 @@ export default class ArgumentsParser { }) .option('verbose', { - group: groupDebug, + group: groupHelp, alias: 'v', description: 'Enable verbose logging, can specify twice (-vv)', type: 'count', @@ -317,8 +327,9 @@ export default class ArgumentsParser { ['$0 copy -i ROMs/ -o /media/SDCard/ROMs/ -D --dir-letter -t', 'Copy ROMs to a flash cart and test them'], ]) - // Colorize help output + // Colorize help output .option('help', { + group: groupHelp, alias: 'h', description: 'Show help', type: 'boolean', diff --git a/src/modules/candidateGenerator.ts b/src/modules/candidateGenerator.ts index b8f971216..e64b6971e 100644 --- a/src/modules/candidateGenerator.ts +++ b/src/modules/candidateGenerator.ts @@ -31,13 +31,14 @@ export default class CandidateGenerator { return output; } + await this.progressBar.setSymbol(Symbols.GENERATING); + await this.progressBar.reset(dat.getParents().length); + + // TODO(cemmer): use filesize combined with CRC for indexing // TODO(cemmer): ability to index files by some other property such as name const crc32ToInputFiles = await CandidateGenerator.indexFilesByCrc(inputRomFiles); await this.progressBar.logInfo(`${dat.getName()}: ${crc32ToInputFiles.size} unique ROM CRC32s found`); - await this.progressBar.setSymbol(Symbols.GENERATING); - await this.progressBar.reset(dat.getParents().length); - // TODO(cemmer): ability to work without DATs, generating a parent/game/release per file // For each parent, try to generate a parent candidate /* eslint-disable no-await-in-loop */ @@ -79,20 +80,25 @@ export default class CandidateGenerator { private static async indexFilesByCrc(files: File[]): Promise> { return files.reduce(async (accPromise, file) => { const acc = await accPromise; - if (acc.has(await file.getCrc32())) { - // Have already seen file, prefer non-archived files - const existing = acc.get(await file.getCrc32()) as File; - if (!(file instanceof ArchiveEntry) && existing instanceof ArchiveEntry) { - acc.set(await file.getCrc32(), file); - } - } else { - // Haven't seen file yet, store it - acc.set(await file.getCrc32(), file); - } + this.addToIndex(acc, await file.getCrc32(), file); + this.addToIndex(acc, await file.getCrc32WithoutHeader(), file); return acc; }, Promise.resolve(new Map())); } + private static addToIndex(map: Map, hash: string, file: File): void { + if (map.has(hash)) { + // Have already seen file, prefer non-archived files + const existing = map.get(hash) as File; + if (!(file instanceof ArchiveEntry) && existing instanceof ArchiveEntry) { + map.set(hash, file); + } + } else { + // Haven't seen file yet, store it + map.set(hash, file); + } + } + private async buildReleaseCandidateForRelease( game: Game, release: Release | undefined, diff --git a/src/modules/headerProcessor.ts b/src/modules/headerProcessor.ts new file mode 100644 index 000000000..127f3c994 --- /dev/null +++ b/src/modules/headerProcessor.ts @@ -0,0 +1,60 @@ +import async, { AsyncResultCallback } from 'async'; + +import ProgressBar, { Symbols } from '../console/progressBar.js'; +import Constants from '../constants.js'; +import File from '../types/files/file.js'; +import FileHeader from '../types/files/fileHeader.js'; +import Options from '../types/options.js'; + +// TODO(cemmer): put debug statements in here +export default class HeaderProcessor { + private readonly options: Options; + + private readonly progressBar: ProgressBar; + + constructor(options: Options, progressBar: ProgressBar) { + this.options = options; + this.progressBar = progressBar; + } + + async process(inputRomFiles: File[]): Promise { + await this.progressBar.logInfo('Processing file headers'); + + await this.progressBar.setSymbol(Symbols.HASHING); + await this.progressBar.reset(inputRomFiles.length); + + return async.mapLimit( + inputRomFiles, + Constants.ROM_HEADER_HASHER_THREADS, + async (inputFile, callback: AsyncResultCallback) => { + await this.progressBar.increment(); + + // Don't do anything if the file already has a header + if (inputFile.getFileHeader()) { + return callback(null, inputFile); + } + + // Can get FileHeader from extension, use that + const headerForExtension = FileHeader.getForFilename(inputFile.getExtractedFilePath()); + if (headerForExtension) { + const fileWithHeader = await inputFile.withFileHeader(headerForExtension).resolve(); + return callback(null, fileWithHeader); + } + + // Should get FileHeader from File, try to + if (this.options.shouldReadFileForHeader(inputFile.getExtractedFilePath())) { + const headerForFile = await inputFile + .extract(async (localFile) => FileHeader.getForFileContents(localFile)); + if (headerForFile) { + const fileWithHeader = await inputFile.withFileHeader(headerForFile).resolve(); + return callback(null, fileWithHeader); + } + await this.progressBar.logWarn(`Couldn't detect header for ${inputFile.toString()}`); + } + + // Should not get FileHeader + return callback(null, inputFile); + }, + ); + } +} diff --git a/src/modules/romScanner.ts b/src/modules/romScanner.ts index 51fd17fad..b5c21a538 100644 --- a/src/modules/romScanner.ts +++ b/src/modules/romScanner.ts @@ -12,7 +12,6 @@ import Scanner from './scanner.js'; * This class will not be run concurrently with any other class. */ export default class ROMScanner extends Scanner { - // TODO(cemmer): support for headered ROM files (e.g. NES) async scan(): Promise { await this.progressBar.setSymbol(Symbols.SEARCHING); await this.progressBar.reset(0); diff --git a/src/modules/romWriter.ts b/src/modules/romWriter.ts index d21ac1ee8..50ca9f263 100644 --- a/src/modules/romWriter.ts +++ b/src/modules/romWriter.ts @@ -6,9 +6,9 @@ import path from 'path'; import ProgressBar, { Symbols } from '../console/progressBar.js'; import Constants from '../constants.js'; import fsPoly from '../polyfill/fsPoly.js'; +import Zip from '../types/archives/zip.js'; import ArchiveEntry from '../types/files/archiveEntry.js'; import File from '../types/files/file.js'; -import Zip from '../types/files/zip.js'; import DAT from '../types/logiqx/dat.js'; import Parent from '../types/logiqx/parent.js'; import ROM from '../types/logiqx/rom.js'; @@ -107,8 +107,10 @@ export default class ROMWriter { const crcToRoms = releaseCandidate.getRomsByCrc32(); return releaseCandidate.getFiles().reduce(async (accPromise, inputFile) => { + // TODO(cemmer): use filesize combined with CRC for indexing const acc = await accPromise; - const rom = crcToRoms.get(await inputFile.getCrc32()) as ROM; + const rom = crcToRoms.get(await inputFile.getCrc32()) + || crcToRoms.get(await inputFile.getCrc32WithoutHeader()) as ROM; let outputFile: File; if (this.options.shouldZip(rom.getName())) { @@ -221,9 +223,11 @@ export default class ROMWriter { // If the file in the output zip already exists and has the same CRC then do nothing const existingOutputEntry = outputZip.getEntry(outputRomFile.getEntryPath() as string); - if (existingOutputEntry?.header.crc === parseInt(await outputRomFile.getCrc32(), 16)) { - await this.progressBar.logDebug(`${outputZipPath}: ${outputRomFile.getEntryPath()} already exists`); - return false; + if (existingOutputEntry) { + if (existingOutputEntry.header.crc === parseInt(await outputRomFile.getCrc32(), 16)) { + await this.progressBar.logDebug(`${outputZipPath}: ${outputRomFile.getEntryPath()} already exists`); + return false; + } } // Write the entry diff --git a/src/modules/scanner.ts b/src/modules/scanner.ts index eb0cdeb78..388ada329 100644 --- a/src/modules/scanner.ts +++ b/src/modules/scanner.ts @@ -1,5 +1,5 @@ import ProgressBar from '../console/progressBar.js'; -import ArchiveFactory from '../types/files/archiveFactory.js'; +import ArchiveFactory from '../types/archives/archiveFactory.js'; import File from '../types/files/file.js'; import Options from '../types/options.js'; diff --git a/src/polyfill/fsPoly.ts b/src/polyfill/fsPoly.ts index 23dafd7ca..c228df562 100644 --- a/src/polyfill/fsPoly.ts +++ b/src/polyfill/fsPoly.ts @@ -77,7 +77,7 @@ export default class FsPoly { fs.rmdirSync(pathLike, options); } else { // Added in: v14.14.0 - fs.rmSync(pathLike, { recursive: true, force: true }); + fs.rmSync(pathLike, { ...options, force: true }); } } else { // Added in: v0.1.21 diff --git a/src/types/archives/archive.ts b/src/types/archives/archive.ts new file mode 100644 index 000000000..1a0a98430 --- /dev/null +++ b/src/types/archives/archive.ts @@ -0,0 +1,21 @@ +import ArchiveEntry from '../files/archiveEntry.js'; + +export default abstract class Archive { + private readonly filePath: string; + + constructor(filePath: string) { + this.filePath = filePath; + } + + getFilePath(): string { + return this.filePath; + } + + abstract getArchiveEntries(): Promise; + + abstract extractEntry( + archiveEntry: ArchiveEntry, + tempDir: string, + callback: (localFile: string) => (T | Promise), + ): Promise; +} diff --git a/src/types/files/archiveFactory.ts b/src/types/archives/archiveFactory.ts similarity index 100% rename from src/types/files/archiveFactory.ts rename to src/types/archives/archiveFactory.ts diff --git a/src/types/files/rar.ts b/src/types/archives/rar.ts similarity index 76% rename from src/types/files/rar.ts rename to src/types/archives/rar.ts index fbdd94c17..9aee1d453 100644 --- a/src/types/files/rar.ts +++ b/src/types/archives/rar.ts @@ -1,11 +1,8 @@ -import { promises as fsPromises } from 'fs'; import unrar from 'node-unrar-js'; import path from 'path'; -import Constants from '../../constants.js'; -import fsPoly from '../../polyfill/fsPoly.js'; +import ArchiveEntry from '../files/archiveEntry.js'; import Archive from './archive.js'; -import ArchiveEntry from './archiveEntry.js'; export default class Rar extends Archive { static readonly SUPPORTED_EXTENSIONS = ['.rar']; @@ -24,9 +21,9 @@ export default class Rar extends Archive { async extractEntry( archiveEntry: ArchiveEntry, + tempDir: string, callback: (localFile: string) => (T | Promise), ): Promise { - const tempDir = await fsPromises.mkdtemp(Constants.GLOBAL_TEMP_DIR); const localFile = path.join(tempDir, archiveEntry.getEntryPath() as string); const rar = await unrar.createExtractorFromFile({ @@ -40,10 +37,6 @@ export default class Rar extends Archive { files: [archiveEntry.getEntryPath()], }).files]; - try { - return await callback(localFile); - } finally { - fsPoly.rmSync(tempDir, { recursive: true }); - } + return callback(localFile); } } diff --git a/src/types/files/sevenZip.ts b/src/types/archives/sevenZip.ts similarity index 84% rename from src/types/files/sevenZip.ts rename to src/types/archives/sevenZip.ts index e4b660fa2..0b80d5c0a 100644 --- a/src/types/files/sevenZip.ts +++ b/src/types/archives/sevenZip.ts @@ -1,12 +1,9 @@ import _7z, { Result } from '7zip-min'; import { Mutex } from 'async-mutex'; -import { promises as fsPromises } from 'fs'; import path from 'path'; -import Constants from '../../constants.js'; -import fsPoly from '../../polyfill/fsPoly.js'; +import ArchiveEntry from '../files/archiveEntry.js'; import Archive from './archive.js'; -import ArchiveEntry from './archiveEntry.js'; export default class SevenZip extends Archive { // p7zip `7za i` @@ -56,9 +53,9 @@ export default class SevenZip extends Archive { async extractEntry( archiveEntry: ArchiveEntry, + tempDir: string, callback: (localFile: string) => (T | Promise), ): Promise { - const tempDir = await fsPromises.mkdtemp(Constants.GLOBAL_TEMP_DIR); const localFile = path.join(tempDir, archiveEntry.getEntryPath()); await new Promise((resolve, reject) => { @@ -71,10 +68,6 @@ export default class SevenZip extends Archive { }); }); - try { - return await callback(localFile); - } finally { - fsPoly.rmSync(tempDir, { recursive: true }); - } + return callback(localFile); } } diff --git a/src/types/files/zip.ts b/src/types/archives/zip.ts similarity index 74% rename from src/types/files/zip.ts rename to src/types/archives/zip.ts index 338c984df..ff9cf42a1 100644 --- a/src/types/files/zip.ts +++ b/src/types/archives/zip.ts @@ -1,11 +1,8 @@ import AdmZip, { IZipEntry } from 'adm-zip'; -import { promises as fsPromises } from 'fs'; import path from 'path'; -import Constants from '../../constants.js'; -import fsPoly from '../../polyfill/fsPoly.js'; +import ArchiveEntry from '../files/archiveEntry.js'; import Archive from './archive.js'; -import ArchiveEntry from './archiveEntry.js'; export default class Zip extends Archive { static readonly SUPPORTED_EXTENSIONS = ['.zip']; @@ -23,9 +20,9 @@ export default class Zip extends Archive { async extractEntry( archiveEntry: ArchiveEntry, + tempDir: string, callback: (localFile: string) => (T | Promise), ): Promise { - const tempDir = await fsPromises.mkdtemp(Constants.GLOBAL_TEMP_DIR); const localFile = path.join(tempDir, archiveEntry.getEntryPath()); const zip = new AdmZip(this.getFilePath()); @@ -42,10 +39,6 @@ export default class Zip extends Archive { archiveEntry.getEntryPath(), ); - try { - return await callback(localFile); - } finally { - fsPoly.rmSync(tempDir, { recursive: true }); - } + return callback(localFile); } } diff --git a/src/types/files/archive.ts b/src/types/files/archive.ts deleted file mode 100644 index b450c29de..000000000 --- a/src/types/files/archive.ts +++ /dev/null @@ -1,11 +0,0 @@ -import ArchiveEntry from './archiveEntry.js'; -import File from './file.js'; - -export default abstract class Archive extends File { - abstract getArchiveEntries(): Promise; - - abstract extractEntry( - archiveEntry: ArchiveEntry, - callback: (localFile: string) => (T | Promise), - ): Promise; -} diff --git a/src/types/files/archiveEntry.ts b/src/types/files/archiveEntry.ts index 53fff30dd..c7cefcf4e 100644 --- a/src/types/files/archiveEntry.ts +++ b/src/types/files/archiveEntry.ts @@ -1,23 +1,47 @@ -import Archive from './archive.js'; +import { promises as fsPromises } from 'fs'; + +import Constants from '../../constants.js'; +import fsPoly from '../../polyfill/fsPoly.js'; +import Archive from '../archives/archive.js'; import File from './file.js'; +import FileHeader from './fileHeader.js'; export default class ArchiveEntry extends File { private readonly archive: Archive; private readonly entryPath: string; - constructor(archive: Archive, entryPath: string, crc?: string) { - super(archive.getFilePath(), crc); + constructor(archive: Archive, entryPath: string, crc?: string, fileHeader?: FileHeader) { + super(archive.getFilePath(), crc, fileHeader); this.archive = archive; this.entryPath = entryPath; } + getExtractedFilePath(): string { + return this.entryPath; + } + getEntryPath(): string { return this.entryPath; } async extract(callback: (localFile: string) => (T | Promise)): Promise { - return this.archive.extractEntry(this, callback); + const tempDir = await fsPromises.mkdtemp(Constants.GLOBAL_TEMP_DIR); + + try { + return await this.archive.extractEntry(this, tempDir, callback); + } finally { + fsPoly.rmSync(tempDir, { recursive: true }); + } + } + + withFileHeader(fileHeader: FileHeader): File { + return new ArchiveEntry( + this.archive, + this.entryPath, + undefined, // the old CRC can't be used, a header will change it + fileHeader, + ); } toString(): string { diff --git a/src/types/files/file.ts b/src/types/files/file.ts index 73b69626d..83707a540 100644 --- a/src/types/files/file.ts +++ b/src/types/files/file.ts @@ -1,58 +1,97 @@ import crc32 from 'crc/crc32'; -import fs, { PathLike } from 'fs'; +import fs from 'fs'; import path from 'path'; +import Constants from '../../constants.js'; +import FileHeader from './fileHeader.js'; + export default class File { private readonly filePath: string; private crc32?: Promise; - constructor(filePath: string, crc?: string) { + private crc32WithoutHeader?: Promise; + + private readonly fileHeader?: FileHeader; + + constructor(filePath: string, crc?: string, fileHeader?: FileHeader) { this.filePath = filePath; if (crc) { this.crc32 = Promise.resolve(crc); } + this.fileHeader = fileHeader; } getFilePath(): string { return this.filePath; } + getExtractedFilePath(): string { + return this.filePath; + } + async getCrc32(): Promise { if (!this.crc32) { - this.crc32 = File.calculateCrc32(this.filePath); + this.crc32 = this.calculateCrc32(false); } return (await this.crc32).toLowerCase().padStart(8, '0'); } + async getCrc32WithoutHeader(): Promise { + if (!this.fileHeader) { + return this.getCrc32(); + } + + if (!this.crc32WithoutHeader) { + this.crc32WithoutHeader = this.calculateCrc32(true); + } + return (await this.crc32WithoutHeader).toLowerCase().padStart(8, '0'); + } + + getFileHeader(): FileHeader | undefined { + return this.fileHeader; + } + + // TODO(cemmer): figure out how to eliminate this isZip(): boolean { return path.extname(this.getFilePath()).toLowerCase() === '.zip'; } - private static async calculateCrc32(pathLike: PathLike): Promise { - return new Promise((resolve, reject) => { - const stream = fs.createReadStream(pathLike, { - highWaterMark: 1024 * 1024, // 1MB - }); + private async calculateCrc32(processHeader: boolean): Promise { + return this.extract(async (localFile) => { + // If we're hashing a file with a header, make sure the file actually has the header magic + // string before excluding it + let start = 0; + if (processHeader && this.fileHeader && await this.fileHeader.fileHasHeader(localFile)) { + start = this.fileHeader.dataOffsetBytes; + } - let crc: number; - stream.on('data', (chunk) => { - if (!crc) { - crc = crc32(chunk); - } else { - crc = crc32(chunk, crc); - } - }); - stream.on('end', () => { - resolve((crc || 0).toString(16)); - }); + return new Promise((resolve, reject) => { + const stream = fs.createReadStream(localFile, { + start, + highWaterMark: Constants.FILE_READING_CHUNK_SIZE, + }); + + let crc: number; + stream.on('data', (chunk) => { + if (!crc) { + crc = crc32(chunk); + } else { + crc = crc32(chunk, crc); + } + }); + stream.on('end', () => { + resolve((crc || 0).toString(16)); + }); - stream.on('error', (err) => reject(err)); + stream.on('error', (err) => reject(err)); + }); }); } async resolve(): Promise { await this.getCrc32(); + await this.getCrc32WithoutHeader(); return this; } @@ -60,6 +99,14 @@ export default class File { return callback(this.filePath); } + withFileHeader(fileHeader: FileHeader): File { + return new File( + this.filePath, + undefined, // the old CRC can't be used, a header will change it + fileHeader, + ); + } + /** ************************* * * * Pseudo Built-Ins * @@ -75,6 +122,7 @@ export default class File { return true; } return this.getFilePath() === other.getFilePath() - && await this.getCrc32() === await other.getCrc32(); + && await this.getCrc32() === await other.getCrc32() + && await this.getCrc32WithoutHeader() === await other.getCrc32WithoutHeader(); } } diff --git a/src/types/files/fileHeader.ts b/src/types/files/fileHeader.ts new file mode 100644 index 000000000..e57ca1b86 --- /dev/null +++ b/src/types/files/fileHeader.ts @@ -0,0 +1,109 @@ +import fs from 'fs'; +import path from 'path'; + +import Constants from '../../constants.js'; + +export default class FileHeader { + private static readonly HEADERS: { [key: string]:FileHeader } = { + // http://7800.8bitdev.org/index.php/A78_Header_Specification + 'No-Intro_A7800.xml': new FileHeader(1, '415441524937383030', 128, '.a78'), + + // https://atarigamer.com/lynx/lnxhdrgen + 'No-Intro_LNX.xml': new FileHeader(0, '4C594E58', 64, '.lnx'), + + // https://www.nesdev.org/wiki/INES + 'No-Intro_NES.xml': new FileHeader(0, '4E4553', 16, '.nes'), + + // https://www.nesdev.org/wiki/FDS_file_format + 'No-Intro_FDS.xml': new FileHeader(0, '464453', 16, '.fds'), + }; + + private static readonly MAX_HEADER_LENGTH = Object.values(FileHeader.HEADERS) + .reduce((max, fileHeader) => Math.max( + max, + fileHeader.headerOffsetBytes + fileHeader.headerValue.length / 2, + ), 0); + + readonly headerOffsetBytes: number; + + readonly headerValue: string; + + readonly dataOffsetBytes: number; + + readonly fileExtension: string; + + private constructor( + headerOffsetBytes: number, + headerValue: string, + dataOffset: number, + fileExtension: string, + ) { + this.headerOffsetBytes = headerOffsetBytes; + this.headerValue = headerValue; + this.dataOffsetBytes = dataOffset; + this.fileExtension = fileExtension; + } + + static getForName(headerName: string): FileHeader | undefined { + return this.HEADERS[headerName]; + } + + static getForFilename(filePath: string): FileHeader | undefined { + const headers = Object.values(this.HEADERS); + for (let i = 0; i < headers.length; i += 1) { + const header = headers[i]; + if (header.fileExtension.toLowerCase() === path.extname(filePath).toLowerCase()) { + return header; + } + } + return undefined; + } + + private static async readHeader(filePath: string, start: number, end: number): Promise { + return new Promise((resolve, reject) => { + const stream = fs.createReadStream(filePath, { + start, + end, + highWaterMark: Constants.FILE_READING_CHUNK_SIZE, + }); + + const chunks: Buffer[] = []; + stream.on('data', (chunk) => { + chunks.push(Buffer.from(chunk)); + }); + stream.on('end', () => { + const header = Buffer.concat(chunks).toString('hex'); + resolve(header.toUpperCase()); + }); + + stream.on('error', (err) => reject(err)); + }); + } + + static async getForFileContents(filePath: string): Promise { + const fileHeader = await FileHeader.readHeader(filePath, 0, this.MAX_HEADER_LENGTH); + + const headers = Object.values(this.HEADERS); + for (let i = 0; i < headers.length; i += 1) { + const header = headers[i]; + const headerValue = fileHeader.slice( + header.headerOffsetBytes * 2, + header.headerOffsetBytes * 2 + header.headerValue.length, + ); + if (headerValue === header.headerValue) { + return header; + } + } + + return undefined; + } + + async fileHasHeader(filePath: string): Promise { + const header = await FileHeader.readHeader( + filePath, + this.headerOffsetBytes, + this.headerOffsetBytes + this.headerValue.length / 2 - 1, + ); + return header.toUpperCase() === this.headerValue.toUpperCase(); + } +} diff --git a/src/types/logiqx/clrMamePro.ts b/src/types/logiqx/clrMamePro.ts index 81e61e663..dc13a8f56 100644 --- a/src/types/logiqx/clrMamePro.ts +++ b/src/types/logiqx/clrMamePro.ts @@ -2,26 +2,44 @@ import 'reflect-metadata'; import { Expose } from 'class-transformer'; +interface ClrMameProOptions { + readonly header?: string; + readonly forceMerging?: 'none' | 'split' | 'full'; + readonly forceNoDump?: 'obsolete' | 'required' | 'ignore'; + readonly forcePacking?: 'zip' | 'unzip'; +} + /** * "CMPro data files use a 'clrmamepro' element to specify details such as the * emulator name, description, category and the data file version." * * @see http://www.logiqx.com/DatFAQs/CMPro.php */ -export default class ClrMamePro { +export default class ClrMamePro implements ClrMameProOptions { + /** + * No-Intro DATs use this to indicate what file header has been added before the raw ROM data. + * {@link FileHeader.HEADERS} + */ @Expose({ name: 'header' }) - private readonly header?: string; + readonly header: string; /** * "To force CMPro to use a particular merging format (none/split/full). Only * do this if the emulator doesn't allow all three of the modes!" */ @Expose({ name: 'forcemerging' }) - private readonly forceMerging: 'none' | 'split' | 'full' = 'split'; + readonly forceMerging: 'none' | 'split' | 'full'; @Expose({ name: 'forcenodump' }) - private readonly forceNoDump: 'obsolete' | 'required' | 'ignore' = 'obsolete'; + readonly forceNoDump: 'obsolete' | 'required' | 'ignore'; @Expose({ name: 'forcepacking' }) - private readonly forcePacking: 'zip' | 'unzip' = 'zip'; + readonly forcePacking: 'zip' | 'unzip'; + + constructor(options?: ClrMameProOptions) { + this.header = options?.header || ''; + this.forceMerging = options?.forceMerging || 'split'; + this.forceNoDump = options?.forceNoDump || 'obsolete'; + this.forcePacking = options?.forcePacking || 'zip'; + } } diff --git a/src/types/logiqx/dat.ts b/src/types/logiqx/dat.ts index 7353677a3..873c119cf 100644 --- a/src/types/logiqx/dat.ts +++ b/src/types/logiqx/dat.ts @@ -59,10 +59,6 @@ export default class DAT { return this.header; } - getName(): string { - return this.getHeader().getName(); - } - getGames(): Game[] { if (this.game instanceof Array) { return this.game; @@ -78,29 +74,29 @@ export default class DAT { // Computed getters + getName(): string { + return this.getHeader().getName(); + } + getNameShort(): string { return this.getName() - // Prefixes + // Prefixes .replace('Non-Redump', '') .replace('Source Code', '') .replace('Unofficial', '') - // Suffixes + // Suffixes .replace('Datfile', '') - .replace('(BigEndian)', '') .replace('(CDN)', '') - .replace('(Decrypted)', '') .replace('(Deprecated)', '') .replace('(Digital)', '') .replace('(Download Play)', '') - .replace('(Headered)', '') .replace('(Misc)', '') - // .replace('(Multiboot)', '') .replace(/\(Parent-Clone\)/g, '') .replace('(PSN)', '') .replace('(Split DLC)', '') .replace('(WAD)', '') .replace('(WIP)', '') - // Cleanup + // Cleanup .replace(/^[ -]+/, '') .replace(/[ -]+$/, '') .replace(/ +/g, ' ') diff --git a/src/types/logiqx/header.ts b/src/types/logiqx/header.ts index ab7c04b81..50f60ce57 100644 --- a/src/types/logiqx/header.ts +++ b/src/types/logiqx/header.ts @@ -6,41 +6,41 @@ import ClrMamePro from './clrMamePro.js'; import RomCenter from './romCenter.js'; interface HeaderOptions { - name?: string; - description?: string; - category?: string; - version?: string; - date?: string; - author?: string; - email?: string; - homepage?: string; - url?: string; - comment?: string; - clrMamePro?: ClrMamePro; - romCenter?: RomCenter; + readonly name?: string; + readonly description?: string; + readonly category?: string; + readonly version?: string; + readonly date?: string; + readonly author?: string; + readonly email?: string; + readonly homepage?: string; + readonly url?: string; + readonly comment?: string; + readonly clrMamePro?: ClrMamePro; + readonly romCenter?: RomCenter; } -export default class Header { +export default class Header implements HeaderOptions { /** * "Name of the emulator without a version number. This field is used by the * update feature of the CMPro profiler." */ @Expose({ name: 'name' }) - private readonly name!: string; + readonly name: string; /** * "Name of the emulator with a version number. This is the name displayed by * CMPro." */ @Expose({ name: 'description' }) - private readonly description!: string; + readonly description: string; /** * "General comment about the emulator (e.g. the systems or game types it * supports)." */ @Expose({ name: 'category' }) - private readonly category?: string; + readonly category?: string; /** * "Version number of the data file. I would recommend using something like a @@ -48,62 +48,56 @@ export default class Header { * be sorted and is unambiguous)." */ @Expose({ name: 'version' }) - private readonly version!: string; + readonly version: string; @Expose({ name: 'date' }) - private readonly date?: string; + readonly date?: string; /** * "Your name and e-mail/web address." */ @Expose({ name: 'author' }) - private readonly author!: string; + readonly author: string; @Expose({ name: 'email' }) - private readonly email?: string; + readonly email?: string; @Expose({ name: 'homepage' }) - private readonly homepage?: string; + readonly homepage?: string; @Expose({ name: 'url' }) - private readonly url?: string; + readonly url?: string; @Expose({ name: 'comment' }) - private readonly comment?: string; + readonly comment?: string; @Type(() => ClrMamePro) @Expose({ name: 'clrmamepro' }) - private readonly clrMamePro?: ClrMamePro; + readonly clrMamePro?: ClrMamePro; @Type(() => RomCenter) @Expose({ name: 'romcenter' }) - private readonly romCenter?: RomCenter; + readonly romCenter?: RomCenter; constructor(options?: HeaderOptions) { - if (options) { - this.name = options.name || ''; - this.description = options.description || ''; - this.category = options.category; - this.version = options.version || ''; - this.date = options.date; - this.author = options.author || ''; - this.email = options.email; - this.homepage = options.homepage; - this.url = options.url; - this.comment = options.comment; - this.clrMamePro = options.clrMamePro; - this.romCenter = options.romCenter; - } + this.name = options?.name || ''; + this.description = options?.description || ''; + this.category = options?.category; + this.version = options?.version || ''; + this.date = options?.date; + this.author = options?.author || ''; + this.email = options?.email; + this.homepage = options?.homepage; + this.url = options?.url; + this.comment = options?.comment; + this.clrMamePro = options?.clrMamePro; + this.romCenter = options?.romCenter; } getName(): string { return this.name; } - getDescription(): string { - return this.description; - } - getVersion(): string { return this.version; } @@ -111,4 +105,8 @@ export default class Header { getDate(): string | undefined { return this.date; } + + getClrMamePro(): ClrMamePro | undefined { + return this.clrMamePro; + } } diff --git a/src/types/logiqx/parent.ts b/src/types/logiqx/parent.ts index f7968c10e..87bb4c950 100644 --- a/src/types/logiqx/parent.ts +++ b/src/types/logiqx/parent.ts @@ -23,18 +23,4 @@ export default class Parent { addChild(child: Game): void { this.games.push(child); } - - // Computed getters - - isBios(): boolean { - return this.getGames().some((game) => game.isBios()); - } - - isRetail(): boolean { - return this.getGames().some((game) => game.isRetail()); - } - - isPrototype(): boolean { - return !this.isRetail() && this.getGames().some((game) => game.isPrototype()); - } } diff --git a/src/types/options.ts b/src/types/options.ts index 371c64c44..f6466eb2b 100644 --- a/src/types/options.ts +++ b/src/types/options.ts @@ -20,6 +20,7 @@ export interface OptionsProps { readonly input?: string[], readonly inputExclude?: string[], readonly output?: string, + readonly header?: string, readonly dirMirror?: boolean, readonly dirDatName?: boolean, readonly dirLetter?: boolean, @@ -63,6 +64,8 @@ export default class Options implements OptionsProps { readonly output!: string; + readonly header!: string; + readonly dirMirror!: boolean; readonly dirDatName!: boolean; @@ -121,14 +124,13 @@ export default class Options implements OptionsProps { readonly help!: boolean; - private tempDir!: string; - constructor(options?: OptionsProps) { this.commands = options?.commands || []; this.dat = options?.dat || []; this.input = options?.input || []; this.inputExclude = options?.inputExclude || []; this.output = options?.output || ''; + this.header = options?.header || ''; this.dirMirror = options?.dirMirror || false; this.dirDatName = options?.dirDatName || false; this.dirLetter = options?.dirLetter || false; @@ -158,39 +160,18 @@ export default class Options implements OptionsProps { this.noBad = options?.noBad || false; this.verbose = options?.verbose || 0; this.help = options?.help || false; - - this.createTempDir(); - this.validate(); } static fromObject(obj: object): Options { return plainToInstance(Options, obj, { enableImplicitConversion: true, - }) - .createTempDir() - .validate(); + }); } toString(): string { return JSON.stringify(instanceToPlain(this)); } - private createTempDir(): Options { - this.tempDir = fsPoly.mkdtempSync(); - process.on('SIGINT', () => { - fsPoly.rmSync(this.tempDir, { - force: true, - recursive: true, - }); - }); - return this; - } - - private validate(): Options { - // TODO(cemmer): validate fields on the class - return this; - } - // Commands private getCommands(): string[] { @@ -211,7 +192,10 @@ export default class Options implements OptionsProps { shouldZip(filePath: string): boolean { return this.getCommands().indexOf('zip') !== -1 - && (!this.getZipExclude() || !micromatch.isMatch(filePath, this.getZipExclude())); + && (!this.getZipExclude() || !micromatch.isMatch( + filePath.replace(/^.[\\/]/, ''), + this.getZipExclude(), + )); } shouldClean(): boolean { @@ -301,7 +285,7 @@ export default class Options implements OptionsProps { } getOutput(dat?: DAT, inputRomPath?: string, romName?: string): string { - let output = this.shouldWrite() ? this.output : this.getTempDir(); + let output = this.shouldWrite() ? this.output : Constants.GLOBAL_TEMP_DIR; if (this.getDirMirror() && inputRomPath) { const mirroredDir = path.dirname(inputRomPath) .replace(/[\\/]/g, path.sep) @@ -337,12 +321,23 @@ export default class Options implements OptionsProps { return path.join( output, `${Constants.COMMAND_NAME}_${moment().format()}.txt` - // Make the filename Windows legal + // Make the filename Windows legal .replace(/:/g, ';') .replace(/[<>:"/\\|?*]/g, '_'), ); } + private getHeader(): string { + return this.header; + } + + shouldReadFileForHeader(filePath: string): boolean { + return this.getHeader().length > 0 && micromatch.isMatch( + filePath.replace(/^.[\\/]/, ''), + this.getHeader(), + ); + } + getDirMirror(): boolean { return this.dirMirror; } @@ -464,10 +459,6 @@ export default class Options implements OptionsProps { return this.help; } - getTempDir(): string { - return this.tempDir; - } - static filterUniqueUpper(array: string[]): string[] { return array .map((value) => value.toUpperCase()) diff --git a/test/console/logger.test.ts b/test/console/logger.test.ts index fcd943f81..b1b6d279a 100644 --- a/test/console/logger.test.ts +++ b/test/console/logger.test.ts @@ -56,7 +56,7 @@ function testLogLevelsAtOrBelow( describe('setLogLevel_getLogLevel', () => { const logLevels = Object.keys(LogLevel).map((ll) => LogLevel[ll as keyof typeof LogLevel]); - test.each(logLevels)('should reflect %s', (logLevel) => { + test.each(logLevels)('should reflect: %s', (logLevel) => { const logger = new Logger(-1, new PassThrough()); expect(logger.getLogLevel()).not.toEqual(logLevel); @@ -66,13 +66,13 @@ describe('setLogLevel_getLogLevel', () => { }); describe('newLine', () => { - testLogLevelsAbove(LogLevel.NEVER - 1)('should not write %s', async (logLevel) => { + testLogLevelsAbove(LogLevel.NEVER - 1)('should not write: %s', async (logLevel) => { const spy = new LoggerSpy(logLevel); spy.getLogger().newLine(); await expect(spy.getOutput()).resolves.toEqual(''); }); - testLogLevelsAtOrBelow(LogLevel.NEVER - 1)('should write %s', async (logLevel) => { + testLogLevelsAtOrBelow(LogLevel.NEVER - 1)('should write: %s', async (logLevel) => { const spy = new LoggerSpy(logLevel); spy.getLogger().newLine(); await expect(spy.getOutput()).resolves.toEqual('\n'); @@ -80,13 +80,13 @@ describe('newLine', () => { }); describe('debug', () => { - testLogLevelsAbove(LogLevel.DEBUG)('should not write %s', async (logLevel) => { + testLogLevelsAbove(LogLevel.DEBUG)('should not write: %s', async (logLevel) => { const spy = new LoggerSpy(logLevel); spy.getLogger().debug('debug message'); await expect(spy.getOutput()).resolves.toEqual(''); }); - testLogLevelsAtOrBelow(LogLevel.DEBUG)('should write %s', async (logLevel) => { + testLogLevelsAtOrBelow(LogLevel.DEBUG)('should write: %s', async (logLevel) => { const spy = new LoggerSpy(logLevel); spy.getLogger().debug('debug message'); await expect(spy.getOutput()).resolves.toContain('debug message'); @@ -94,13 +94,13 @@ describe('debug', () => { }); describe('info', () => { - testLogLevelsAbove(LogLevel.INFO)('should not write %s', async (logLevel) => { + testLogLevelsAbove(LogLevel.INFO)('should not write: %s', async (logLevel) => { const spy = new LoggerSpy(logLevel); spy.getLogger().info('info message'); await expect(spy.getOutput()).resolves.toEqual(''); }); - testLogLevelsAtOrBelow(LogLevel.INFO)('should write %s', async (logLevel) => { + testLogLevelsAtOrBelow(LogLevel.INFO)('should write: %s', async (logLevel) => { const spy = new LoggerSpy(logLevel); spy.getLogger().info('info message'); await expect(spy.getOutput()).resolves.toContain('info message'); @@ -108,13 +108,13 @@ describe('info', () => { }); describe('warn', () => { - testLogLevelsAbove(LogLevel.WARN)('should not write %s', async (logLevel) => { + testLogLevelsAbove(LogLevel.WARN)('should not write: %s', async (logLevel) => { const spy = new LoggerSpy(logLevel); spy.getLogger().warn('warn message'); await expect(spy.getOutput()).resolves.toEqual(''); }); - testLogLevelsAtOrBelow(LogLevel.WARN)('should write %s', async (logLevel) => { + testLogLevelsAtOrBelow(LogLevel.WARN)('should write: %s', async (logLevel) => { const spy = new LoggerSpy(logLevel); spy.getLogger().warn('warn message'); await expect(spy.getOutput()).resolves.toContain('warn message'); @@ -122,13 +122,13 @@ describe('warn', () => { }); describe('error', () => { - testLogLevelsAbove(LogLevel.ERROR)('should not write %s', async (logLevel) => { + testLogLevelsAbove(LogLevel.ERROR)('should not write: %s', async (logLevel) => { const spy = new LoggerSpy(logLevel); spy.getLogger().error('error message'); await expect(spy.getOutput()).resolves.toEqual(''); }); - testLogLevelsAtOrBelow(LogLevel.ERROR)('should write %s', async (logLevel) => { + testLogLevelsAtOrBelow(LogLevel.ERROR)('should write: %s', async (logLevel) => { const spy = new LoggerSpy(logLevel); spy.getLogger().error('error message'); await expect(spy.getOutput()).resolves.toContain('error message'); @@ -136,13 +136,13 @@ describe('error', () => { }); describe('printHeader', () => { - testLogLevelsAbove(LogLevel.NEVER - 1)('should not write %s', async (logLevel) => { + testLogLevelsAbove(LogLevel.NEVER - 1)('should not write: %s', async (logLevel) => { const spy = new LoggerSpy(logLevel); spy.getLogger().printHeader(); await expect(spy.getOutput()).resolves.toEqual(''); }); - testLogLevelsAtOrBelow(LogLevel.NEVER - 1)('should write %s', async (logLevel) => { + testLogLevelsAtOrBelow(LogLevel.NEVER - 1)('should write: %s', async (logLevel) => { const spy = new LoggerSpy(logLevel); spy.getLogger().printHeader(); await expect(spy.getOutput()).resolves.not.toEqual(''); diff --git a/test/fixtures/roms/7z/fizzbuzz.7z b/test/fixtures/roms/7z/fizzbuzz.7z index 6be6e539d..63c032f4b 100644 Binary files a/test/fixtures/roms/7z/fizzbuzz.7z and b/test/fixtures/roms/7z/fizzbuzz.7z differ diff --git a/test/fixtures/roms/7z/foobar.7z b/test/fixtures/roms/7z/foobar.7z index 8433bdfa5..43abfa7d4 100644 Binary files a/test/fixtures/roms/7z/foobar.7z and b/test/fixtures/roms/7z/foobar.7z differ diff --git a/test/fixtures/roms/fizzbuzz.zip b/test/fixtures/roms/fizzbuzz.zip index 4f2b20b79..d469d9c23 100644 Binary files a/test/fixtures/roms/fizzbuzz.zip and b/test/fixtures/roms/fizzbuzz.zip differ diff --git a/test/fixtures/roms/foobar.rom b/test/fixtures/roms/foobar.lnx similarity index 100% rename from test/fixtures/roms/foobar.rom rename to test/fixtures/roms/foobar.lnx diff --git a/test/fixtures/roms/headered/LCDTestROM.lnx.rar b/test/fixtures/roms/headered/LCDTestROM.lnx.rar new file mode 100644 index 000000000..82117f899 Binary files /dev/null and b/test/fixtures/roms/headered/LCDTestROM.lnx.rar differ diff --git a/test/fixtures/roms/headered/allpads.nes b/test/fixtures/roms/headered/allpads.nes new file mode 100644 index 000000000..b35f27775 Binary files /dev/null and b/test/fixtures/roms/headered/allpads.nes differ diff --git a/test/fixtures/roms/headered/color_test.nintendoentertainmentsystem b/test/fixtures/roms/headered/color_test.nintendoentertainmentsystem new file mode 100644 index 000000000..65d5dc36e Binary files /dev/null and b/test/fixtures/roms/headered/color_test.nintendoentertainmentsystem differ diff --git a/test/fixtures/roms/headered/diagnostic_test_cartridge.a78.7z b/test/fixtures/roms/headered/diagnostic_test_cartridge.a78.7z new file mode 100644 index 000000000..aa818e9c0 Binary files /dev/null and b/test/fixtures/roms/headered/diagnostic_test_cartridge.a78.7z differ diff --git a/test/fixtures/roms/headered/fds_joypad_test.fds.zip b/test/fixtures/roms/headered/fds_joypad_test.fds.zip new file mode 100644 index 000000000..4512cb1e3 Binary files /dev/null and b/test/fixtures/roms/headered/fds_joypad_test.fds.zip differ diff --git a/test/fixtures/roms/rar/fizzbuzz.rar b/test/fixtures/roms/rar/fizzbuzz.rar index c6e3373af..581faf2a0 100644 Binary files a/test/fixtures/roms/rar/fizzbuzz.rar and b/test/fixtures/roms/rar/fizzbuzz.rar differ diff --git a/test/fixtures/roms/rar/foobar.rar b/test/fixtures/roms/rar/foobar.rar index 10ae0714b..17692f1f4 100644 Binary files a/test/fixtures/roms/rar/foobar.rar and b/test/fixtures/roms/rar/foobar.rar differ diff --git a/test/fixtures/roms/raw/fizzbuzz.rom b/test/fixtures/roms/raw/fizzbuzz.nes similarity index 100% rename from test/fixtures/roms/raw/fizzbuzz.rom rename to test/fixtures/roms/raw/fizzbuzz.nes diff --git a/test/fixtures/roms/raw/foobar.rom b/test/fixtures/roms/raw/foobar.lnx similarity index 100% rename from test/fixtures/roms/raw/foobar.rom rename to test/fixtures/roms/raw/foobar.lnx diff --git a/test/fixtures/roms/zip/fizzbuzz.zip b/test/fixtures/roms/zip/fizzbuzz.zip index 4f2b20b79..d469d9c23 100644 Binary files a/test/fixtures/roms/zip/fizzbuzz.zip and b/test/fixtures/roms/zip/fizzbuzz.zip differ diff --git a/test/fixtures/roms/zip/foobar.zip b/test/fixtures/roms/zip/foobar.zip index 23e77cdc5..d951ee6cc 100644 Binary files a/test/fixtures/roms/zip/foobar.zip and b/test/fixtures/roms/zip/foobar.zip differ diff --git a/test/igir.test.ts b/test/igir.test.ts index 58daa678f..a289cda8c 100644 --- a/test/igir.test.ts +++ b/test/igir.test.ts @@ -1,28 +1,31 @@ import { jest } from '@jest/globals'; +import fg from 'fast-glob'; import fs from 'fs'; import path from 'path'; import Logger from '../src/console/logger.js'; import LogLevel from '../src/console/logLevel.js'; +import Constants from '../src/constants.js'; import Igir from '../src/igir.js'; import fsPoly from '../src/polyfill/fsPoly.js'; import Options, { OptionsProps } from '../src/types/options.js'; const LOGGER = new Logger(LogLevel.NEVER); -async function expectEndToEnd(options: OptionsProps, expectedFiles: string[]): Promise { +async function expectEndToEnd(optionsProps: OptionsProps, expectedFiles: string[]): Promise { const tempInput = fsPoly.mkdtempSync(); fsPoly.copyDirSync('./test/fixtures', tempInput); const tempOutput = fsPoly.mkdtempSync(); - await new Igir(new Options({ + const options = new Options({ dat: [path.join(tempInput, 'dats', '*')], input: [path.join(tempInput, 'roms', '**', '*')], - ...options, + ...optionsProps, output: tempOutput, verbose: Number.MAX_SAFE_INTEGER, - }), LOGGER).main(); + }); + await new Igir(options, LOGGER).main(); const writtenRoms = fs.readdirSync(tempOutput); @@ -34,6 +37,12 @@ async function expectEndToEnd(options: OptionsProps, expectedFiles: string[]): P fsPoly.rmSync(tempInput, { recursive: true }); fsPoly.rmSync(tempOutput, { recursive: true }); + + const reports = await fg(path.join( + path.dirname(options.getOutputReport()), + `${Constants.COMMAND_NAME}_*.txt`, + )); + reports.forEach((report) => fsPoly.rmSync(report)); } jest.setTimeout(10_000); @@ -80,7 +89,6 @@ it('should copy and clean', async () => { }); it('should report without copy', async () => { - // TODO(cemmer): cleanup the written report await expectEndToEnd({ commands: ['report'], }, []); diff --git a/test/modules/argumentsParser.test.ts b/test/modules/argumentsParser.test.ts index 20ea596b0..fca9e77e0 100644 --- a/test/modules/argumentsParser.test.ts +++ b/test/modules/argumentsParser.test.ts @@ -131,6 +131,13 @@ describe('options', () => { expect(argumentsParser.parse(['copy', '--input', os.devNull, '--output', 'foo', '--output', 'bar']).getOutput()).toEqual('bar'); }); + it('should parse "header"', () => { + expect(() => argumentsParser.parse([...dummyCommandAndRequiredArgs, '-H'])).toThrow(/not enough arguments/i); + expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '-H', '**/*']).shouldReadFileForHeader('file.rom')).toEqual(true); + expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--header', '**/*']).shouldReadFileForHeader('file.rom')).toEqual(true); + expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--header', '**/*', '--header', 'nope']).shouldReadFileForHeader('file.rom')).toEqual(false); + }); + it('should parse "dir-mirror"', () => { expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dir-mirror']).getDirMirror()).toEqual(true); expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dir-mirror', 'true']).getDirMirror()).toEqual(true); @@ -391,5 +398,6 @@ describe('options', () => { it('should parse "help"', () => { expect(argumentsParser.parse(['-h']).getHelp()).toEqual(true); expect(argumentsParser.parse(['--help']).getHelp()).toEqual(true); + expect(argumentsParser.parse(['--help', '100']).getHelp()).toEqual(true); }); }); diff --git a/test/modules/candidateFilter.test.ts b/test/modules/candidateFilter.test.ts index c10d98014..78533f23a 100644 --- a/test/modules/candidateFilter.test.ts +++ b/test/modules/candidateFilter.test.ts @@ -9,8 +9,8 @@ import Options, { OptionsProps } from '../../src/types/options.js'; import ReleaseCandidate from '../../src/types/releaseCandidate.js'; import ProgressBarFake from '../console/progressBarFake.js'; -function buildCandidateFilter(options: object = {}): CandidateFilter { - return new CandidateFilter(Options.fromObject(options), new ProgressBarFake()); +function buildCandidateFilter(options: OptionsProps = {}): CandidateFilter { + return new CandidateFilter(new Options(options), new ProgressBarFake()); } async function expectFilteredCandidates( diff --git a/test/modules/candidateGenerator.test.ts b/test/modules/candidateGenerator.test.ts index 48c2f86e2..5f97b6e5c 100644 --- a/test/modules/candidateGenerator.test.ts +++ b/test/modules/candidateGenerator.test.ts @@ -1,7 +1,7 @@ import CandidateGenerator from '../../src/modules/candidateGenerator.js'; +import Zip from '../../src/types/archives/zip.js'; import ArchiveEntry from '../../src/types/files/archiveEntry.js'; import File from '../../src/types/files/file.js'; -import Zip from '../../src/types/files/zip.js'; import DAT from '../../src/types/logiqx/dat.js'; import Game from '../../src/types/logiqx/game.js'; import Header from '../../src/types/logiqx/header.js'; diff --git a/test/modules/datScanner.test.ts b/test/modules/datScanner.test.ts index 9649db9b9..215605a03 100644 --- a/test/modules/datScanner.test.ts +++ b/test/modules/datScanner.test.ts @@ -8,7 +8,7 @@ import ProgressBarFake from '../console/progressBarFake.js'; jest.setTimeout(10_000); function createDatScanner(dat: string[]): DATScanner { - return new DATScanner(Options.fromObject({ dat }), new ProgressBarFake()); + return new DATScanner(new Options({ dat }), new ProgressBarFake()); } it('should throw on nonexistent paths', async () => { diff --git a/test/modules/headerProcessor.test.ts b/test/modules/headerProcessor.test.ts new file mode 100644 index 000000000..74f4cdf92 --- /dev/null +++ b/test/modules/headerProcessor.test.ts @@ -0,0 +1,76 @@ +import HeaderProcessor from '../../src/modules/headerProcessor.js'; +import ROMScanner from '../../src/modules/romScanner.js'; +import Options from '../../src/types/options.js'; +import ProgressBarFake from '../console/progressBarFake.js'; + +describe('extension has possible header', () => { + it('should do nothing if extension not found', async () => { + const inputRomFiles = await new ROMScanner(new Options({ + input: ['./test/fixtures/roms/{,**/}*.rom'], + }), new ProgressBarFake()).scan(); + expect(inputRomFiles.length).toBeGreaterThan(0); + + const processedRomFiles = await new HeaderProcessor(new Options(), new ProgressBarFake()) + .process(inputRomFiles); + + expect(processedRomFiles).toHaveLength(inputRomFiles.length); + /* eslint-disable no-await-in-loop */ + for (let i = 0; i < processedRomFiles.length; i += 1) { + await expect(inputRomFiles[i].equals(processedRomFiles[i])).resolves.toEqual(true); + } + }); + + it('should process headered files', async () => { + const inputRomFiles = await new ROMScanner(new Options({ + input: ['./test/fixtures/roms/headered/*{.a78,.lnx,.nes,.fds}*'], + }), new ProgressBarFake()).scan(); + expect(inputRomFiles.length).toBeGreaterThan(0); + + const processedRomFiles = await new HeaderProcessor(new Options(), new ProgressBarFake()) + .process(inputRomFiles); + + expect(processedRomFiles).toHaveLength(inputRomFiles.length); + /* eslint-disable no-await-in-loop */ + for (let i = 0; i < processedRomFiles.length; i += 1) { + // CRC should have changed + await expect(inputRomFiles[i].equals(processedRomFiles[i])).resolves.toEqual(false); + } + }); +}); + +describe('should read file for header', () => { + it('should do nothing with un-headered files', async () => { + const inputRomFiles = await new ROMScanner(new Options({ + input: ['./test/fixtures/roms/!(headered){,/}*'], + }), new ProgressBarFake()).scan(); + expect(inputRomFiles.length).toBeGreaterThan(0); + + const processedRomFiles = await new HeaderProcessor(new Options({ + header: '**/*', + }), new ProgressBarFake()).process(inputRomFiles); + + expect(processedRomFiles).toHaveLength(inputRomFiles.length); + /* eslint-disable no-await-in-loop */ + for (let i = 0; i < processedRomFiles.length; i += 1) { + await expect(inputRomFiles[i].equals(processedRomFiles[i])).resolves.toEqual(true); + } + }); + + it('should process headered files', async () => { + const inputRomFiles = await new ROMScanner(new Options({ + input: ['./test/fixtures/roms/headered/!(*{.a78,.lnx,.nes,.fds}*)'], + }), new ProgressBarFake()).scan(); + expect(inputRomFiles.length).toBeGreaterThan(0); + + const processedRomFiles = await new HeaderProcessor(new Options({ + header: '**/*', + }), new ProgressBarFake()).process(inputRomFiles); + + expect(processedRomFiles).toHaveLength(inputRomFiles.length); + /* eslint-disable no-await-in-loop */ + for (let i = 0; i < processedRomFiles.length; i += 1) { + // CRC should have changed + await expect(inputRomFiles[i].equals(processedRomFiles[i])).resolves.toEqual(false); + } + }); +}); diff --git a/test/modules/outputCleaner.test.ts b/test/modules/outputCleaner.test.ts index ee6b86191..a1d91f5fa 100644 --- a/test/modules/outputCleaner.test.ts +++ b/test/modules/outputCleaner.test.ts @@ -9,10 +9,12 @@ import ProgressBarFake from '../console/progressBarFake.js'; jest.setTimeout(10_000); +const romFixtures = path.join('test', 'fixtures', 'roms'); + async function runOutputCleaner(writtenFilePathsToExclude: string[]): Promise { // Copy the fixture files to a temp directory const tempDir = fsPoly.mkdtempSync(); - fsPoly.copyDirSync('./test/fixtures', tempDir); + fsPoly.copyDirSync(romFixtures, tempDir); const writtenRomFilesToExclude = writtenFilePathsToExclude .map((filePath) => new File(path.join(tempDir, filePath), '00000000')); @@ -38,30 +40,32 @@ async function runOutputCleaner(writtenFilePathsToExclude: string[]): Promise { - const existingFiles = fsPoly.walkSync('./test/fixtures') - .map((filePath) => filePath.replace(/^test[\\/]fixtures[\\/]/, '')); + const existingFiles = fsPoly.walkSync(romFixtures) + .map((filePath) => filePath.replace(/^test[\\/]fixtures[\\/]roms[\\/]/, '')) + .sort(); const filesRemaining = await runOutputCleaner([]); expect(filesRemaining).toEqual(existingFiles); }); it('should delete nothing if all match', async () => { - const existingFiles = fsPoly.walkSync('./test/fixtures') - .map((filePath) => filePath.replace(/^test[\\/]fixtures[\\/]/, '')); + const existingFiles = fsPoly.walkSync(romFixtures) + .map((filePath) => filePath.replace(/^test[\\/]fixtures[\\/]roms[\\/]/, '')) + .sort(); const filesRemaining = await runOutputCleaner(existingFiles); expect(filesRemaining).toEqual(existingFiles); }); it('should delete some if some matched', async () => { const filesRemaining = await runOutputCleaner([ - path.join('roms', '7z', 'empty.7z'), - path.join('roms', 'raw', 'fizzbuzz.rom'), - path.join('roms', 'zip', 'foobar.zip'), + path.join('7z', 'empty.7z'), + path.join('raw', 'fizzbuzz.nes'), + path.join('zip', 'foobar.zip'), 'non-existent file', ]); expect(filesRemaining).toEqual([ - path.join('roms', '7z', 'empty.7z'), - path.join('roms', 'raw', 'fizzbuzz.rom'), - path.join('roms', 'zip', 'foobar.zip'), + path.join('7z', 'empty.7z'), + path.join('raw', 'fizzbuzz.nes'), + path.join('zip', 'foobar.zip'), ]); }); diff --git a/test/modules/reportGenerator.test.ts b/test/modules/reportGenerator.test.ts new file mode 100644 index 000000000..d28a79909 --- /dev/null +++ b/test/modules/reportGenerator.test.ts @@ -0,0 +1,2 @@ +// TODO(cemmer) +it('ok', () => {}); diff --git a/test/modules/romScanner.test.ts b/test/modules/romScanner.test.ts index da8a820bf..ef047d0cf 100644 --- a/test/modules/romScanner.test.ts +++ b/test/modules/romScanner.test.ts @@ -8,7 +8,7 @@ import ProgressBarFake from '../console/progressBarFake.js'; jest.setTimeout(10_000); function createRomScanner(input: string[], inputExclude: string[] = []): ROMScanner { - return new ROMScanner(Options.fromObject({ input, inputExclude }), new ProgressBarFake()); + return new ROMScanner(new Options({ input, inputExclude }), new ProgressBarFake()); } it('should throw on nonexistent paths', async () => { @@ -25,22 +25,33 @@ it('should return empty list on no results', async () => { await expect(createRomScanner([os.devNull]).scan()).resolves.toEqual([]); }); -it('should return empty list when input matches inputExclude', async () => { - // TODO(cemmer) -}); - it('should not throw on bad archives', async () => { await expect(createRomScanner(['test/fixtures/roms/**/invalid.zip']).scan()).resolves.toHaveLength(0); await expect(createRomScanner(['test/fixtures/roms/**/invalid.rar']).scan()).resolves.toHaveLength(0); await expect(createRomScanner(['test/fixtures/roms/**/invalid.7z']).scan()).resolves.toHaveLength(0); }); -it('should scan multiple files', async () => { - const expectedRomFiles = 22; - await expect(createRomScanner(['test/fixtures/roms']).scan()).resolves.toHaveLength(expectedRomFiles); - await expect(createRomScanner(['test/fixtures/roms/*', 'test/fixtures/roms/**/*.{rom,zip,rar,7z}']).scan()).resolves.toHaveLength(expectedRomFiles); - await expect(createRomScanner(['test/fixtures/roms/**/*.{rom,zip,rar,7z}']).scan()).resolves.toHaveLength(expectedRomFiles); - await expect(createRomScanner(['test/fixtures/roms/**/*.{rom,zip,rar,7z}', 'test/fixtures/roms/**/*.{rom,zip,rar,7z}']).scan()).resolves.toHaveLength(expectedRomFiles); +describe('multiple files', () => { + it('no files are excluded', async () => { + const expectedRomFiles = 27; + await expect(createRomScanner(['test/fixtures/roms']).scan()).resolves.toHaveLength(expectedRomFiles); + await expect(createRomScanner(['test/fixtures/roms/*', 'test/fixtures/roms/**/*']).scan()).resolves.toHaveLength(expectedRomFiles); + await expect(createRomScanner(['test/fixtures/roms/**/*']).scan()).resolves.toHaveLength(expectedRomFiles); + await expect(createRomScanner(['test/fixtures/roms/**/*', 'test/fixtures/roms/**/*.{rom,zip}']).scan()).resolves.toHaveLength(expectedRomFiles); + }); + + it('some files are excluded', async () => { + await expect(createRomScanner(['test/fixtures/roms/**/*'], ['test/fixtures/roms/**/*.rom']).scan()).resolves.toHaveLength(23); + await expect(createRomScanner(['test/fixtures/roms/**/*'], ['test/fixtures/roms/**/*.rom', 'test/fixtures/roms/**/*.rom']).scan()).resolves.toHaveLength(23); + await expect(createRomScanner(['test/fixtures/roms/**/*'], ['test/fixtures/roms/**/*.rom', 'test/fixtures/roms/**/*.zip']).scan()).resolves.toHaveLength(17); + }); + + it('all files are excluded', async () => { + await expect(createRomScanner(['test/fixtures/roms/**/*'], ['test/fixtures/roms/**/*']).scan()).resolves.toEqual([]); + await expect(createRomScanner(['test/fixtures/roms/**/*'], ['test/fixtures/roms/**/*', 'test/fixtures/roms/**/*']).scan()).resolves.toEqual([]); + await expect(createRomScanner(['test/fixtures/roms/**/*'], ['test/fixtures/roms/*', 'test/fixtures/roms/*/**/*']).scan()).resolves.toEqual([]); + await expect(createRomScanner(['test/fixtures/roms/**/*'], ['test/fixtures/roms/**/*.zip', 'test/fixtures/roms/**/*']).scan()).resolves.toEqual([]); + }); }); it('should scan single files', async () => { diff --git a/test/modules/romWriter.test.ts b/test/modules/romWriter.test.ts index 92b967401..f7de9178b 100644 --- a/test/modules/romWriter.test.ts +++ b/test/modules/romWriter.test.ts @@ -6,7 +6,6 @@ import path from 'path'; import ROMScanner from '../../src/modules/romScanner.js'; import ROMWriter from '../../src/modules/romWriter.js'; import fsPoly from '../../src/polyfill/fsPoly.js'; -import ArchiveEntry from '../../src/types/files/archiveEntry.js'; import File from '../../src/types/files/file.js'; import DAT from '../../src/types/logiqx/dat.js'; import Game from '../../src/types/logiqx/game.js'; @@ -71,10 +70,7 @@ async function indexFilesByName( const releaseCandidates = await Promise.all(romFiles .map(async (romFile) => { const release = new Release(romName, 'UNK', undefined); - let romFileName = romFile.getFilePath(); - if (romFile instanceof ArchiveEntry) { - romFileName = (romFile).getEntryPath(); - } + const romFileName = romFile.getExtractedFilePath(); const rom = new ROM( path.basename(romFileName), await romFile.getCrc32(), @@ -176,7 +172,7 @@ describe('zip', () => { const inputFiles = fsPoly.walkSync(inputTemp); expect(inputFiles.length).toBeGreaterThan(0); - const parentsToCandidates = await indexFilesByName(inputTemp, '**/*'); + const parentsToCandidates = await indexFilesByName(inputTemp, '**/!(headered)/*'); // Write once const firstWrittenPaths = await runRomWriter(outputTemp, { @@ -213,7 +209,7 @@ describe('zip', () => { const inputFiles = fsPoly.walkSync(inputTemp); expect(inputFiles.length).toBeGreaterThan(0); - const parentsToCandidates = await indexFilesByName(inputTemp, '**/*'); + const parentsToCandidates = await indexFilesByName(inputTemp, '**/!(headered)/*'); // Write once const firstWrittenPaths = await runRomWriter(outputTemp, { @@ -245,7 +241,7 @@ describe('zip', () => { const inputFiles = fsPoly.walkSync(inputTemp); expect(inputFiles.length).toBeGreaterThan(0); - const parentsToCandidates = await indexFilesByName(inputTemp, '**/*'); + const parentsToCandidates = await indexFilesByName(inputTemp, '**/!(headered)/*'); const existingZip = new AdmZip(); existingZip.addFile('something.rom', Buffer.from('something')); @@ -275,7 +271,7 @@ describe('zip', () => { const inputFiles = fsPoly.walkSync(inputTemp); expect(inputFiles.length).toBeGreaterThan(0); - const parentsToCandidates = await indexFilesByName(inputTemp, '**/*'); + const parentsToCandidates = await indexFilesByName(inputTemp, '**/!(headered)/*'); const writtenPaths = await runRomWriter(outputTemp, { commands: ['copy', 'zip', 'test'], @@ -403,8 +399,8 @@ describe('raw', () => { }, parentsToCandidates); expect(writtenPaths).toEqual([ 'empty.rom', - 'fizzbuzz.rom', - 'foobar.rom', + 'fizzbuzz.nes', + 'foobar.lnx', 'loremipsum.rom', 'unknown.rom', ]); @@ -420,7 +416,7 @@ describe('raw', () => { const inputFiles = fsPoly.walkSync(inputTemp); expect(inputFiles.length).toBeGreaterThan(0); - const parentsToCandidates = await indexFilesByName(inputTemp, '**/*'); + const parentsToCandidates = await indexFilesByName(inputTemp, '**/!(headered)/*'); // Write once const firstWrittenPaths = await runRomWriter(outputTemp, { @@ -428,8 +424,8 @@ describe('raw', () => { }, parentsToCandidates); expect(firstWrittenPaths).toEqual([ 'empty.rom', - 'fizzbuzz.rom', - 'foobar.rom', + 'fizzbuzz.nes', + 'foobar.lnx', 'loremipsum.rom', 'unknown.rom', ]); @@ -440,8 +436,8 @@ describe('raw', () => { }, parentsToCandidates); expect(secondWrittenPaths).toEqual([ 'empty.rom', - 'fizzbuzz.rom', - 'foobar.rom', + 'fizzbuzz.nes', + 'foobar.lnx', 'loremipsum.rom', 'unknown.rom', ]); @@ -457,7 +453,7 @@ describe('raw', () => { const inputFiles = fsPoly.walkSync(inputTemp); expect(inputFiles.length).toBeGreaterThan(0); - const parentsToCandidates = await indexFilesByName(inputTemp, '**/*'); + const parentsToCandidates = await indexFilesByName(inputTemp, '**/!(headered)/*'); // Write once const firstWrittenPaths = await runRomWriter(outputTemp, { @@ -465,8 +461,8 @@ describe('raw', () => { }, parentsToCandidates); expect(firstWrittenPaths).toEqual([ 'empty.rom', - 'fizzbuzz.rom', - 'foobar.rom', + 'fizzbuzz.nes', + 'foobar.lnx', 'loremipsum.rom', 'unknown.rom', ]); @@ -489,15 +485,15 @@ describe('raw', () => { const inputFiles = fsPoly.walkSync(inputTemp); expect(inputFiles.length).toBeGreaterThan(0); - const parentsToCandidates = await indexFilesByName(inputTemp, '**/*'); + const parentsToCandidates = await indexFilesByName(inputTemp, '**/!(headered)/*'); const writtenPaths = await runRomWriter(outputTemp, { commands: ['copy', 'test'], }, parentsToCandidates); expect(writtenPaths).toEqual([ 'empty.rom', - 'fizzbuzz.rom', - 'foobar.rom', + 'fizzbuzz.nes', + 'foobar.lnx', 'loremipsum.rom', 'unknown.rom', ]); @@ -519,8 +515,8 @@ describe('raw', () => { commands: ['copy', 'test'], }, parentsToCandidates); expect(writtenPaths).toEqual([ - 'fizzbuzz.rom', - 'foobar.rom', + 'fizzbuzz.nes', + 'foobar.lnx', 'loremipsum.rom', 'unknown.rom', ]); @@ -543,8 +539,8 @@ describe('raw', () => { }, parentsToCandidates); expect(writtenPaths).toEqual([ 'empty.rom', - 'fizzbuzz.rom', - 'foobar.rom', + 'fizzbuzz.nes', + 'foobar.lnx', 'loremipsum.rom', 'unknown.rom', ]); @@ -566,8 +562,8 @@ describe('raw', () => { commands: ['copy', 'test'], }, parentsToCandidates); expect(writtenPaths).toEqual([ - 'fizzbuzz.rom', - 'foobar.rom', + 'fizzbuzz.nes', + 'foobar.lnx', 'loremipsum.rom', 'unknown.rom', ]); @@ -591,8 +587,8 @@ describe('raw', () => { }, parentsToCandidates); expect(writtenPaths).toEqual([ 'empty.rom', - 'fizzbuzz.rom', - 'foobar.rom', + 'fizzbuzz.nes', + 'foobar.lnx', 'loremipsum.rom', 'unknown.rom', ]); diff --git a/test/modules/statusGenerator.test.ts b/test/modules/statusGenerator.test.ts new file mode 100644 index 000000000..d28a79909 --- /dev/null +++ b/test/modules/statusGenerator.test.ts @@ -0,0 +1,2 @@ +// TODO(cemmer) +it('ok', () => {}); diff --git a/test/types/files/archive.test.ts b/test/types/files/archive.test.ts index 1bba0c38e..06b3b1685 100644 --- a/test/types/files/archive.test.ts +++ b/test/types/files/archive.test.ts @@ -1,16 +1,16 @@ -import ArchiveFactory from '../../../src/types/files/archiveFactory.js'; +import ArchiveFactory from '../../../src/types/archives/archiveFactory.js'; describe('getArchiveEntries', () => { // TODO(cemmer): fixture archives with multiple entries test.each([ // fizzbuzz - ['./test/fixtures/roms/7z/fizzbuzz.7z', 'fizzbuzz.rom', '370517b5'], - ['./test/fixtures/roms/rar/fizzbuzz.rar', 'fizzbuzz.rom', '370517b5'], - ['./test/fixtures/roms/zip/fizzbuzz.zip', 'fizzbuzz.rom', '370517b5'], + ['./test/fixtures/roms/7z/fizzbuzz.7z', 'fizzbuzz.nes', '370517b5'], + ['./test/fixtures/roms/rar/fizzbuzz.rar', 'fizzbuzz.nes', '370517b5'], + ['./test/fixtures/roms/zip/fizzbuzz.zip', 'fizzbuzz.nes', '370517b5'], // foobar - ['./test/fixtures/roms/7z/foobar.7z', 'foobar.rom', 'b22c9747'], - ['./test/fixtures/roms/rar/foobar.rar', 'foobar.rom', 'b22c9747'], - ['./test/fixtures/roms/zip/foobar.zip', 'foobar.rom', 'b22c9747'], + ['./test/fixtures/roms/7z/foobar.7z', 'foobar.lnx', 'b22c9747'], + ['./test/fixtures/roms/rar/foobar.rar', 'foobar.lnx', 'b22c9747'], + ['./test/fixtures/roms/zip/foobar.zip', 'foobar.lnx', 'b22c9747'], // loremipsum ['./test/fixtures/roms/7z/loremipsum.7z', 'loremipsum.rom', '70856527'], ['./test/fixtures/roms/rar/loremipsum.rar', 'loremipsum.rom', '70856527'], @@ -19,7 +19,7 @@ describe('getArchiveEntries', () => { ['./test/fixtures/roms/7z/unknown.7z', 'unknown.rom', '377a7727'], ['./test/fixtures/roms/rar/unknown.rar', 'unknown.rom', '377a7727'], ['./test/fixtures/roms/zip/unknown.zip', 'unknown.rom', '377a7727'], - ])('should enumerate the single file archive %s', async (filePath, expectedEntryPath, expectedCrc) => { + ])('should enumerate the single file archive: %s', async (filePath, expectedEntryPath, expectedCrc) => { const archive = ArchiveFactory.archiveFrom(filePath); const entries = await archive.getArchiveEntries(); diff --git a/test/types/files/archiveEntry.test.ts b/test/types/files/archiveEntry.test.ts index d9b8db372..f4ce4b57e 100644 --- a/test/types/files/archiveEntry.test.ts +++ b/test/types/files/archiveEntry.test.ts @@ -2,10 +2,11 @@ import fs from 'fs'; import ROMScanner from '../../../src/modules/romScanner.js'; import fsPoly from '../../../src/polyfill/fsPoly.js'; +import ArchiveFactory from '../../../src/types/archives/archiveFactory.js'; +import SevenZip from '../../../src/types/archives/sevenZip.js'; +import Zip from '../../../src/types/archives/zip.js'; import ArchiveEntry from '../../../src/types/files/archiveEntry.js'; -import ArchiveFactory from '../../../src/types/files/archiveFactory.js'; -import SevenZip from '../../../src/types/files/sevenZip.js'; -import Zip from '../../../src/types/files/zip.js'; +import FileHeader from '../../../src/types/files/fileHeader.js'; import Options from '../../../src/types/options.js'; import ProgressBarFake from '../../console/progressBarFake.js'; @@ -20,59 +21,111 @@ describe('getEntryPath', () => { }); }); -describe('extract', () => { - it('should extract zip files', async () => { - // Note: this will only return valid zips with at least one file - const zips = await new ROMScanner(new Options({ - input: ['./test/fixtures/roms/zip'], - }), new ProgressBarFake()).scan(); - expect(zips).toHaveLength(4); +describe('getCrc32', () => { + test.each([ + ['./test/fixtures/roms/7z/fizzbuzz.7z', '370517b5'], + ['./test/fixtures/roms/rar/fizzbuzz.rar', '370517b5'], + ['./test/fixtures/roms/zip/fizzbuzz.zip', '370517b5'], + ['./test/fixtures/roms/7z/foobar.7z', 'b22c9747'], + ['./test/fixtures/roms/rar/foobar.rar', 'b22c9747'], + ['./test/fixtures/roms/zip/foobar.zip', 'b22c9747'], + ['./test/fixtures/roms/7z/loremipsum.7z', '70856527'], + ['./test/fixtures/roms/rar/loremipsum.rar', '70856527'], + ['./test/fixtures/roms/zip/loremipsum.zip', '70856527'], + ['./test/fixtures/roms/headered/diagnostic_test_cartridge.a78.7z', 'f6cc9b1c'], + ['./test/fixtures/roms/headered/fds_joypad_test.fds.zip', '1e58456d'], + ['./test/fixtures/roms/headered/LCDTestROM.lnx.rar', '2d251538'], + ])('should hash the full archive entry: %s', async (filePath, expectedCrc) => { + const archive = ArchiveFactory.archiveFrom(filePath); - const temp = fsPoly.mkdtempSync(); - /* eslint-disable no-await-in-loop */ - for (let i = 0; i < zips.length; i += 1) { - const zip = zips[i]; - await zip.extract((localFile) => { - expect(fs.existsSync(localFile)).toEqual(true); - expect(localFile).not.toEqual(zip.getFilePath()); - }); - } - fsPoly.rmSync(temp, { recursive: true }); + const archiveEntries = await archive.getArchiveEntries(); + expect(archiveEntries).toHaveLength(1); + const archiveEntry = archiveEntries[0]; + + await expect(archiveEntry.getCrc32()).resolves.toEqual(expectedCrc); }); +}); - it('should extract rar files', async () => { - // Note: this will only return valid rars with at least one file - const rars = await new ROMScanner(new Options({ - input: ['./test/fixtures/roms/rar'], - }), new ProgressBarFake()).scan(); - expect(rars).toHaveLength(4); +describe('getCrc32WithoutHeader', () => { + test.each([ + ['./test/fixtures/roms/7z/fizzbuzz.7z', '370517b5'], + ['./test/fixtures/roms/rar/fizzbuzz.rar', '370517b5'], + ['./test/fixtures/roms/zip/fizzbuzz.zip', '370517b5'], + ['./test/fixtures/roms/7z/foobar.7z', 'b22c9747'], + ['./test/fixtures/roms/rar/foobar.rar', 'b22c9747'], + ['./test/fixtures/roms/zip/foobar.zip', 'b22c9747'], + ['./test/fixtures/roms/7z/loremipsum.7z', '70856527'], + ['./test/fixtures/roms/rar/loremipsum.rar', '70856527'], + ['./test/fixtures/roms/zip/loremipsum.zip', '70856527'], + ['./test/fixtures/roms/headered/diagnostic_test_cartridge.a78.7z', 'f6cc9b1c'], + ['./test/fixtures/roms/headered/fds_joypad_test.fds.zip', '1e58456d'], + ['./test/fixtures/roms/headered/LCDTestROM.lnx.rar', '2d251538'], + ])('should hash the full archive entry when no header given: %s', async (filePath, expectedCrc) => { + const archive = ArchiveFactory.archiveFrom(filePath); - const temp = fsPoly.mkdtempSync(); - /* eslint-disable no-await-in-loop */ - for (let i = 0; i < rars.length; i += 1) { - const rar = rars[i]; - await rar.extract((localFile) => { - expect(fs.existsSync(localFile)).toEqual(true); - expect(localFile).not.toEqual(rar.getFilePath()); - }); - } - fsPoly.rmSync(temp, { recursive: true }); + const archiveEntries = await archive.getArchiveEntries(); + expect(archiveEntries).toHaveLength(1); + const archiveEntry = archiveEntries[0]; + + await expect(archiveEntry.getCrc32WithoutHeader()).resolves.toEqual(expectedCrc); + }); + + test.each([ + ['./test/fixtures/roms/7z/fizzbuzz.7z', '370517b5'], + ['./test/fixtures/roms/rar/fizzbuzz.rar', '370517b5'], + ['./test/fixtures/roms/zip/fizzbuzz.zip', '370517b5'], + ['./test/fixtures/roms/7z/foobar.7z', 'b22c9747'], + ['./test/fixtures/roms/rar/foobar.rar', 'b22c9747'], + ['./test/fixtures/roms/zip/foobar.zip', 'b22c9747'], + ])('should hash the full archive entry when header is given but not present in file: %s', async (filePath, expectedCrc) => { + const archive = ArchiveFactory.archiveFrom(filePath); + + const archiveEntries = await archive.getArchiveEntries(); + expect(archiveEntries).toHaveLength(1); + const archiveEntry = archiveEntries[0].withFileHeader( + FileHeader.getForFilename(archiveEntries[0].getExtractedFilePath()) as FileHeader, + ); + + await expect(archiveEntry.getCrc32WithoutHeader()).resolves.toEqual(expectedCrc); }); - it('should extract 7z files', async () => { - // Note: this will only return valid 7z's with at least one file - const sevenZips = await new ROMScanner(new Options({ - input: ['./test/fixtures/roms/7z'], + test.each([ + ['./test/fixtures/roms/headered/diagnostic_test_cartridge.a78.7z', 'a1eaa7c1'], + ['./test/fixtures/roms/headered/fds_joypad_test.fds.zip', '3ecbac61'], + ['./test/fixtures/roms/headered/LCDTestROM.lnx.rar', '42583855'], + ])('should hash the archive entry without the header when header is given and present in file: %s', async (filePath, expectedCrc) => { + const archive = ArchiveFactory.archiveFrom(filePath); + + const archiveEntries = await archive.getArchiveEntries(); + expect(archiveEntries).toHaveLength(1); + const archiveEntry = archiveEntries[0].withFileHeader( + FileHeader.getForFilename(archiveEntries[0].getExtractedFilePath()) as FileHeader, + ); + + await expect(archiveEntry.getCrc32()).resolves.not.toEqual(expectedCrc); + await expect(archiveEntry.getCrc32WithoutHeader()).resolves.toEqual(expectedCrc); + }); +}); + +describe('extract', () => { + it('should extract archived files', async () => { + // Note: this will only return valid archives with at least one file + const archiveEntries = await new ROMScanner(new Options({ + input: [ + './test/fixtures/roms/zip', + './test/fixtures/roms/rar', + './test/fixtures/roms/7z', + ], }), new ProgressBarFake()).scan(); - expect(sevenZips).toHaveLength(4); + expect(archiveEntries).toHaveLength(12); const temp = fsPoly.mkdtempSync(); /* eslint-disable no-await-in-loop */ - for (let i = 0; i < sevenZips.length; i += 1) { - const sevenZip = sevenZips[i]; - await sevenZip.extract((localFile) => { + for (let i = 0; i < archiveEntries.length; i += 1) { + const zip = archiveEntries[i]; + await zip.extract((localFile) => { expect(fs.existsSync(localFile)).toEqual(true); - expect(localFile).not.toEqual(sevenZip.getFilePath()); + expect(localFile).not.toEqual(zip.getFilePath()); }); } fsPoly.rmSync(temp, { recursive: true }); diff --git a/test/types/files/file.test.ts b/test/types/files/file.test.ts index be8adf9bb..245d029cb 100644 --- a/test/types/files/file.test.ts +++ b/test/types/files/file.test.ts @@ -2,8 +2,8 @@ import fs from 'fs'; import ROMScanner from '../../../src/modules/romScanner.js'; import fsPoly from '../../../src/polyfill/fsPoly.js'; -import ArchiveFactory from '../../../src/types/files/archiveFactory.js'; import File from '../../../src/types/files/file.js'; +import FileHeader from '../../../src/types/files/fileHeader.js'; import Options from '../../../src/types/options.js'; import ProgressBarFake from '../../console/progressBarFake.js'; @@ -28,21 +28,47 @@ describe('getCrc32', () => { test.each([ ['./test/fixtures/roms/raw/empty.rom', '00000000'], - ['./test/fixtures/roms/raw/fizzbuzz.rom', '370517b5'], - ['./test/fixtures/roms/raw/foobar.rom', 'b22c9747'], + ['./test/fixtures/roms/raw/fizzbuzz.nes', '370517b5'], + ['./test/fixtures/roms/raw/foobar.lnx', 'b22c9747'], ['./test/fixtures/roms/raw/loremipsum.rom', '70856527'], - ])('should hash the file path: %s', async (filePath, expectedCrc) => { + ])('should hash the full file: %s', async (filePath, expectedCrc) => { const file = new File(filePath); await expect(file.getCrc32()).resolves.toEqual(expectedCrc); }); }); +describe('getCrc32WithoutHeader', () => { + test.each([ + ['./test/fixtures/roms/headered/allpads.nes', '9180a163'], + ])('should hash the full file when no header given: %s', async (filePath, expectedCrc) => { + const file = new File(filePath); + await expect(file.getCrc32WithoutHeader()).resolves.toEqual(expectedCrc); + }); + + test.each([ + ['./test/fixtures/roms/raw/fizzbuzz.nes', '370517b5'], + ['./test/fixtures/roms/raw/foobar.lnx', 'b22c9747'], + ])('should hash the full file when header is given but not present in file: %s', async (filePath, expectedCrc) => { + const file = new File(filePath) + .withFileHeader(FileHeader.getForFilename(filePath) as FileHeader); + await expect(file.getCrc32WithoutHeader()).resolves.toEqual(expectedCrc); + }); + + test.each([ + ['./test/fixtures/roms/headered/allpads.nes', '6339abe6'], + ])('should hash the full file when header is given and present in file: %s', async (filePath, expectedCrc) => { + const file = new File(filePath) + .withFileHeader(FileHeader.getForFilename(filePath) as FileHeader); + await expect(file.getCrc32WithoutHeader()).resolves.toEqual(expectedCrc); + }); +}); + describe('isZip', () => { test.each([ './test/fixtures/roms/zip/empty.zip', './test/fixtures/roms/fizzbuzz.zip', ])('should return true when appropriate', (filePath) => { - const file = ArchiveFactory.archiveFrom(filePath); + const file = new File(filePath); expect(file.isZip()).toEqual(true); }); @@ -52,7 +78,7 @@ describe('isZip', () => { './test/fixtures/roms/rar/fizzbuzz.rar', './test/fixtures/roms/unknown.rar', ])('should return false when appropriate', (filePath) => { - const file = ArchiveFactory.archiveFrom(filePath); + const file = new File(filePath); expect(file.isZip()).toEqual(false); }); }); diff --git a/test/types/files/fileHeader.test.ts b/test/types/files/fileHeader.test.ts new file mode 100644 index 000000000..c2e3e9bfb --- /dev/null +++ b/test/types/files/fileHeader.test.ts @@ -0,0 +1,81 @@ +import ROMScanner from '../../../src/modules/romScanner.js'; +import FileHeader from '../../../src/types/files/fileHeader.js'; +import Options from '../../../src/types/options.js'; +import ProgressBarFake from '../../console/progressBarFake.js'; + +describe('getForName', () => { + test.each([ + 'No-Intro_A7800.xml', + 'No-Intro_LNX.xml', + 'No-Intro_NES.xml', + 'No-Intro_FDS.xml', + ])('should get a file header for name: %s', (headerName) => { + const fileHeader = FileHeader.getForName(headerName); + expect(fileHeader).not.toBeUndefined(); + }); + + test.each([ + '', + ' ', + '🤷', + ])('should not get a file header for name: %s', (headerName) => { + const fileHeader = FileHeader.getForName(headerName); + expect(fileHeader).toBeUndefined(); + }); +}); + +describe('getForExtension', () => { + test.each([ + 'rom.a78', + 'rom.lnx', + 'rom.nes', + 'rom.fds', + 'rom.zip.fds', + ])('should get a file header for extension: %s', (filePath) => { + const fileHeader = FileHeader.getForFilename(filePath); + expect(fileHeader).not.toBeUndefined(); + }); + + test.each([ + '', + ' ', + '.nes', + 'rom.zip', + 'rom.nes.zip', + ])('should not get a file header for extension: %s', (filePath) => { + const fileHeader = FileHeader.getForFilename(filePath); + expect(fileHeader).toBeUndefined(); + }); +}); + +describe('getForFile', () => { + it('should get a file header for headered files', async () => { + const headeredRoms = await new ROMScanner(new Options({ + input: ['./test/fixtures/roms/headered'], + }), new ProgressBarFake()).scan(); + expect(headeredRoms).toHaveLength(5); + + /* eslint-disable no-await-in-loop */ + for (let i = 0; i < headeredRoms.length; i += 1) { + await headeredRoms[i].extract(async (localFile) => { + const fileHeader = await FileHeader.getForFileContents(localFile); + expect(fileHeader).not.toBeUndefined(); + }); + } + }); + + it('should not get a file header for dummy files', async () => { + const headeredRoms = await new ROMScanner(new Options({ + input: ['./test/fixtures/roms/!(headered){,/}*'], + }), new ProgressBarFake()).scan(); + expect(headeredRoms.length).toBeGreaterThan(0); + + /* eslint-disable no-await-in-loop */ + for (let i = 0; i < headeredRoms.length; i += 1) { + await headeredRoms[i].extract(async (localFile) => { + const fileHeader = await FileHeader.getForFileContents(localFile); + expect(fileHeader).toBeUndefined(); + }); + } + }); +}); diff --git a/test/types/releaseCandidate.test.ts b/test/types/releaseCandidate.test.ts index 087e1f034..720c0ac03 100644 --- a/test/types/releaseCandidate.test.ts +++ b/test/types/releaseCandidate.test.ts @@ -26,7 +26,7 @@ describe('getRegion', () => { }); describe('getLanguages', () => { - test.each(ReleaseCandidate.getLanguages())('should return the release language; %s', (language) => { + test.each(ReleaseCandidate.getLanguages())('should return the release language: %s', (language) => { const release = new Release('release', 'UNK', language); const releaseCandidate = new ReleaseCandidate(new Game(), release, [], []); expect(releaseCandidate.getLanguages()).toEqual([language]); @@ -37,7 +37,7 @@ describe('getLanguages', () => { ['En,Fr,De', ['EN', 'FR', 'DE']], ['It+En,Fr,De,Es,It,Nl,Sv,Da', ['IT', 'EN', 'FR', 'DE', 'ES', 'NL', 'SV', 'DA']], ['En,Fr,It+Es,It', ['EN', 'FR', 'IT', 'ES']], - ])('should return the language from game name; %s', (languages, expectedLanguages) => { + ])('should return the language from game name: %s', (languages, expectedLanguages) => { const releaseCandidate = new ReleaseCandidate(new Game({ name: `game (${languages})` }), undefined, [], []); expect(releaseCandidate.getLanguages()).toEqual(expectedLanguages); });