romman-cli

#!/usr/bin/env python3

## Romman - utility to compare your console ROMs with accuracy-focused datasheets
## Copyright (c) 2021 moonburnt
##
## This program is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program.  If not, see https://www.gnu.org/licenses/gpl-3.0.txt

import logging
import romman
from sys import exit
import argparse
from os import makedirs, rename
from os.path import join

TOOL_NAME = romman.configuration.TOOL_NAME
LAUNCHER_NAME = f"{TOOL_NAME}-cli"
PROGRAM_DIRECTORY = romman.configuration.PROGRAM_DIRECTORY
CACHE_DIRECTORY = romman.configuration.CACHE_DIRECTORY
DEFAULT_DATASHEETS_DIRECTORY = romman.configuration.DEFAULT_DATASHEETS_DIRECTORY

NOINTRO_PREFIX = romman.configuration.NOINTRO_PREFIX
REDUMP_PREFIX = romman.configuration.REDUMP_PREFIX
TOSEC_PREFIX = romman.configuration.TOSEC_PREFIX
MAME_PREFIX = romman.configuration.MAME_PREFIX

log = logging.getLogger()
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
formatter = logging.Formatter(
                        fmt='[%(asctime)s][%(name)s][%(levelname)s] %(message)s',
                        datefmt='%H:%M:%S',
                        )
handler.setFormatter(formatter)
log.addHandler(handler)

#argparse shenanigans
ap = argparse.ArgumentParser()
ap.add_argument("items", help=(
        "Path to ROM file or directory with ROMs to compare with database. You "
        f"can supply multiple paths at once. If directory - {TOOL_NAME} will "
        "also seek for ROMs in its subdirectories"
                ), nargs='*', type=str)
ap.add_argument("--datfiles", help=(
        "Custom path to datasheet file or directory with datasheets. If used - "
        f"{TOOL_NAME} will compare ROMs with these instead of default datasheets. "
        f"You can supply multiple paths at once. If directory - {TOOL_NAME} will "
        "also seek for datasheets in its subdirectories"
            ), nargs='*', type=str)
ap.add_argument("--debug", help=f"Add debug messages to {TOOL_NAME}'s output",
                           action="store_true")
ap.add_argument("--update-datfiles", help=(
        "Download latest available datasheets. Can be used with provider-specific "
        "prefixes. If no valid prefixes has been received - will batch-download "
        "datasheets from all supported providers. Valid prefixes are the following: "
        f"{NOINTRO_PREFIX}, {REDUMP_PREFIX}, {TOSEC_PREFIX}, {MAME_PREFIX}"
        ), nargs='*', type=str)
ap.add_argument("--allow-rename", help=(
        "Rename ROMs that match datasheet hashes, but have incorrect filenames. "
        "Doesnt affect files inside archives"
        ), action="store_true")
args = ap.parse_args()

if args.debug:
    log.setLevel(logging.DEBUG) #Overriding default value from above

#this is nasty as hell, but thats the best way to ensure that args.update_datfiles
#has been passed even if its empty. Or at least out of solutions I've found
try:
    len(args.update_datfiles)
except:
    pass
else:
    log.info("Updating the database (may take some time)")
    romman.dat_updater.datasheets_updater(args.update_datfiles)
    log.info("Successfully updated the database!")

if not args.items:
    print(f"Got no ROMs to verify! For usage info, see {LAUNCHER_NAME} -h")
    exit(0)

if args.datfiles:
    datasheets = args.datfiles
    log.debug(f"Attempting to load {args.datfiles} instead of default datasheets")
else:
    datasheets = [DEFAULT_DATASHEETS_DIRECTORY]

log.info("Loading the database (may take a while)")
data_files = []
for item in datasheets:
    try:
        df = romman.file_processing.get_files(item)
    except FileNotFoundError as e:
        #ensuring that directory we will reffer to in message below exists
        #I know this looks ugly, maybe will find a better solution later
        if item == DEFAULT_DATASHEETS_DIRECTORY:
            makedirs(DEFAULT_DATASHEETS_DIRECTORY, exist_ok=True)
        else:
            log.warning(f"{item} doesnt exist. Skipping")
            continue
    except Exception as e:
        log.warning(f"Couldnt process datasheets on path {item}: {e}. Skipping")
        continue
    else:
        data_files.extend(df)

if not data_files:
    log.critical(
        "Couldnt find any valid data files! Either place them manually into "
        f"{DEFAULT_DATASHEETS_DIRECTORY} or run {LAUNCHER_NAME} --update-datfiles "
        "to download latest datasheets from supported sources"
        )
    exit(1)

database = []
for item in data_files:
    #this check is meh, but important - applying wrong parser to wrong file worthy
    #hundreds of mbytes may eat all ram
    if item.endswith('.dat'.lower()):
        log.debug(f"Guessing that {item} is standard .dat file")
        parser = romman.data_parsers.dat_file
    elif item.endswith('.xml'.lower()):
        log.debug(f"Guessing that {item} is mame .xml file")
        parser = romman.data_parsers.mame_xml
    else:
        log.warning(f"Couldnt process data file {item}: unknown file format")
        continue

    try:
        data = parser(item)
    except Exception as e:
        log.warning(f"Couldnt process data file {item}: {e}")
        continue
    else:
        database.extend(data)

if not database:
    log.critical(f"Couldnt find any valid database entries! Abort")
    exit(1)

log.info("Getting the filelist from provided arguments")
filepaths = []
for item in args.items:
    try:
        f = romman.file_processing.get_files(item)
    except Exception as e:
        log.warning(f"Couldnt get files from {item}: {e}. Skipping")
        continue
    else:
        filepaths.extend(f)

log.info("Calculating hash sums of provided files")
files = []
for item in filepaths:
    try:
        data = romman.file_processing.file_processor(item)
    except Exception as e:
        log.warning(f"Couldnt get hash of {item}: {e}. Skipping")
        continue
    else:
        files.extend(data)

if not files:
    log.critical(f"No valid file entries has been received! Abort")
    exit(1)

log.debug(f"Got following files to process: {files}")
#This may backfire if some entry match multiple datasheets
#But I never experienced it thus far to be sure
matching_files = [(item, entry) for item in files for entry in database if item['crc'] == entry['crc']]

incorrect_names_counter = 0
renamed_files_counter = 0
for item, entry in matching_files:
    output_name = item['path']
    if item['is_archive']:
        output_name = join(item['location'], item['path'])

    log.info(f"'{output_name}' matches '{entry['name']}' in '{join(entry['category'], entry['group'])}'!")

    if item['name'] != entry['name']:
        incorrect_names_counter += 1
        if args.allow_rename and not item['is_archive']:
            new_name = join(item['location'], entry['name'])
            log.info(f"Renaming '{output_name}' to '{new_name}'")
            #maybe put this on try/except?
            rename(output_name, new_name)
            renamed_files_counter += 1
        elif args.allow_rename and item['is_archive']:
            log.warning(f"'{output_name}' has incorrect name, but is "
                         "part of archive - wont rename")
        else:
            log.warning(f"'{output_name}' has incorrect name, should be '{entry['name']}'")

#Get list of unique filepaths as archive files may contain multiple files
unique_filepaths = []
for file in files:
    if file['is_archive']:
        if file['location'] not in unique_filepaths:
            unique_filepaths.append(file['location'])
    else:
        if file['path'] not in unique_filepaths:
            unique_filepaths.append(file['path'])

matches_counter = len(matching_files)
errors_counter = len(set(filepaths) - set(unique_filepaths))
misses_counter = len(files) - matches_counter
ignored_renames_counter = incorrect_names_counter - renamed_files_counter

print(f"{TOOL_NAME} has finished its job:")
print(f"Got {matches_counter} ROMs matching provided datasheets, {misses_counter} "
      f"non-matching and was unable to process {errors_counter} files.")
if args.allow_rename and incorrect_names_counter:
    print(f"Out of {incorrect_names_counter} files with incorrect names, "
          f"{renamed_files_counter} were renamed and {ignored_renames_counter} skipped.")