index.js

const logger = require('pino')()
const percentile = require('percentile')
const fs = require('fs')
const debug = require('debug')('data-generator')
const path = require('path')

/**
 * Calculate min, max and mean of an array of values
 *
 * @param {Array} arr : array of number
 * @returns {Object} : { max, min, avg, total }
 */
const minMaxMean = (arr) => {
    let max = arr[0]
    let min = typeof arr[0] !== 'number' ? 0 : arr[0]
    let sum = typeof arr[0] !== 'number' ? 0 : arr[0]
    let nbItems = 0
    for (let i = 1; i < arr.length; i++) {
        if (typeof arr[i] !== 'number') continue
        if (arr[i] > max) {
            max = arr[i]
        }
        if (arr[i] < min) {
            min = arr[i]
        }
        sum = sum + arr[i]
        nbItems++
    }
    return { max, min, avg: sum / nbItems, total: nbItems }
}

const getAdditionalInfo = (additionalInfo, currentCmd, metric, packageName) => {
    if (!additionalInfo[currentCmd.title]) additionalInfo[currentCmd.title] = {}
    if (!additionalInfo[currentCmd.title][metric]) {
        additionalInfo[currentCmd.title][metric] = {
            max: currentCmd.budget.fail[metric].value,
            min: currentCmd.budget.fail[metric].value,
            packagesMax: [packageName],
            packagesMin: [packageName]
        }
    } else if (currentCmd.budget.fail[metric].value > additionalInfo[currentCmd.title][metric].max) {
        additionalInfo[currentCmd.title][metric].max = currentCmd.budget.fail[metric].value
        additionalInfo[currentCmd.title][metric].packagesMax = [packageName]
    } else if (currentCmd.budget.fail[metric].value === additionalInfo[currentCmd.title][metric].max) {
        additionalInfo[currentCmd.title][metric].packagesMin.push(packageName)
    } else if (currentCmd.budget.fail[metric].value < additionalInfo[currentCmd.title][metric].min) {
        additionalInfo[currentCmd.title][metric].min = currentCmd.budget.fail[metric].value
        additionalInfo[currentCmd.title][metric].packagesMin = [packageName]
    } else if (currentCmd.budget.fail[metric].value === additionalInfo[currentCmd.title][metric].min) {
        additionalInfo[currentCmd.title][metric].packagesMin.push(packageName)
    }
}

/**
 * Generate the json report
 */
const generateJSON = async () => {
    // read report generated by the crawler
    const rawdata = await fs.promises.readFile(process.env.INPUT)
    const data = JSON.parse(rawdata)

    const metrics = {
        general: {
            npmsFinal: [],
            npmsQuality: [],
            npmsMaintenance: [],
            npmsPopularity: [],
            qualscan: []
        },
        consumption: {
            npm: [],
            host: []
        }
    }
    const qualscanMetrics = {}

    const additionalInfo = {}

    const packages = data.packages
    let nbPackages = 0

    // treat each package results
    for (const packageName in packages) {
        const currentPackage = packages[packageName]
        try {
            currentPackage.qualscan = JSON.parse(currentPackage.qualscan)

            // loop through qualscan plugins results
            for (let i = 0; i < packages[packageName].qualscan.data.cmds.length; i++) {
                const currentCmd = packages[packageName].qualscan.data.cmds[i]
                if (!currentCmd.budget) continue

                // loop through metrics in budget
                for (const metric in currentCmd.budget.fail) {
                    if (!metrics[currentCmd.title]) metrics[currentCmd.title] = {}
                    if (!metrics[currentCmd.title][metric]) metrics[currentCmd.title][metric] = []
                    metrics[currentCmd.title][metric].push(currentCmd.budget.fail[metric].value)

                    getAdditionalInfo(additionalInfo, currentCmd, metric, packageName)
                }

                if (!qualscanMetrics[currentCmd.title]) qualscanMetrics[currentCmd.title] = {}
                if (!qualscanMetrics[currentCmd.title][currentCmd.level]) qualscanMetrics[currentCmd.title][currentCmd.level] = 0
                qualscanMetrics[currentCmd.title][currentCmd.level]++
            }

            // save general metrics
            metrics.general.qualscan.push(currentPackage.qualscan.data.score)
            metrics.general.npmsFinal.push(currentPackage.npms.score.final)
            metrics.general.npmsQuality.push(currentPackage.npms.score.detail.quality)
            metrics.general.npmsMaintenance.push(currentPackage.npms.score.detail.maintenance)
            metrics.general.npmsPopularity.push(currentPackage.npms.score.detail.popularity)

            // handle electrical consumption
            if (currentPackage.consumption) {
                metrics.consumption.npm.push(currentPackage.consumption.npm)
                metrics.consumption.host.push(currentPackage.consumption.host)
            }
            nbPackages++
        } catch (err) {
            delete packages[packageName].qualscan
        }
    }

    debug(require('util').inspect(additionalInfo, { showHidden: false, depth: null }))

    // clean metrics
    for (const cmdName in metrics) {
        const currentCmd = metrics[cmdName]
        for (const metric in currentCmd) {
            currentCmd[metric] = currentCmd[metric].filter((value) => {
                return typeof value === 'number'
            })
        }
    }

    // calculate percentiles of every metrics
    for (const cmdName in metrics) {
        const currentCmd = metrics[cmdName]
        for (const metric in currentCmd) {
            const result = percentile(
                [25, 50, 75, 90, 95, 99],
                currentCmd[metric]
            )
            const minMaxAvg = minMaxMean(currentCmd[metric])
            currentCmd[metric] = {
                ...minMaxAvg,
                percentiles: {
                    25: result[0],
                    50: result[1],
                    75: result[2],
                    90: result[3],
                    95: result[4],
                    99: result[5]
                }
            }
        }
    }

    metrics.qualscanMetrics = qualscanMetrics

    const payload = {
        time: data.time,
        duration: data.duration,
        metrics,
        machine: data.machine,
        skippedPackages: data.skippedPackages,
        nbPackages
    }

    try {
        await fs.promises.writeFile(path.join(process.env.OUTPUT, `${new Date(data.time).toJSON().substr(0, 10)}.json`), JSON.stringify(payload), 'utf8')
        logger.info('Data generated!')
    } catch (err) {
        logger.error(err)
    }
}

generateJSON()