Skip to content

Commit

Permalink
refactor(node)!: Invert the inheritance between Yarn and Npm
Browse files Browse the repository at this point in the history 
Prepare for re-writing large parts of `Npm` to no more rely on the file
hierarchy of `node_modules` when constructing the dependency tree.

While the same is planned for `Yarn` too, this change is just a
refactoring step which allows to do `Npm` first.

Signed-off-by: Frank Viernau <frank_viernau@epam.com>
  • Loading branch information
@fviernau @sschuberth
fviernau authored and sschuberth committed Nov 6, 2024
1 parent 390a055 commit 743fd64
Show file tree
Hide file tree
Showing 4 changed files with 327 additions and 321 deletions.
317 changes: 5 additions & 312 deletions plugins/package-managers/node/src/main/kotlin/Npm.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,68 +22,41 @@
package org.ossreviewtoolkit.plugins.packagemanagers.node

import java.io.File
import java.util.concurrent.ConcurrentHashMap

import org.apache.logging.log4j.kotlin.logger

import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory
import org.ossreviewtoolkit.analyzer.PackageManager
import org.ossreviewtoolkit.analyzer.PackageManager.Companion.getFallbackProjectName
import org.ossreviewtoolkit.analyzer.PackageManager.Companion.processPackageVcs
import org.ossreviewtoolkit.analyzer.PackageManager.Companion.processProjectVcs
import org.ossreviewtoolkit.analyzer.PackageManagerResult
import org.ossreviewtoolkit.downloader.VcsHost
import org.ossreviewtoolkit.downloader.VersionControlSystem
import org.ossreviewtoolkit.model.DependencyGraph
import org.ossreviewtoolkit.model.Hash
import org.ossreviewtoolkit.model.Identifier
import org.ossreviewtoolkit.model.Issue
import org.ossreviewtoolkit.model.Package
import org.ossreviewtoolkit.model.Project
import org.ossreviewtoolkit.model.ProjectAnalyzerResult
import org.ossreviewtoolkit.model.RemoteArtifact
import org.ossreviewtoolkit.model.Severity
import org.ossreviewtoolkit.model.VcsInfo
import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
import org.ossreviewtoolkit.model.config.PackageManagerConfiguration
import org.ossreviewtoolkit.model.config.RepositoryConfiguration
import org.ossreviewtoolkit.model.createAndLogIssue
import org.ossreviewtoolkit.model.readTree
import org.ossreviewtoolkit.model.utils.DependencyGraphBuilder
import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NON_EXISTING_SEMVER
import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NodePackageManager
import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NpmDependencyHandler
import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NpmDetection
import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NpmModuleInfo
import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.expandNpmShortcutUrl
import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.fixNpmDownloadUrl
import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.mapNpmLicenses
import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.parseNpmAuthor
import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.parseNpmVcsInfo
import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.splitNpmNamespaceAndName
import org.ossreviewtoolkit.utils.common.CommandLineTool
import org.ossreviewtoolkit.utils.common.Os
import org.ossreviewtoolkit.utils.common.ProcessCapture
import org.ossreviewtoolkit.utils.common.collectMessages
import org.ossreviewtoolkit.utils.common.fieldNamesOrEmpty
import org.ossreviewtoolkit.utils.common.isSymbolicLink
import org.ossreviewtoolkit.utils.common.realFile
import org.ossreviewtoolkit.utils.common.stashDirectories
import org.ossreviewtoolkit.utils.common.textValueOrEmpty
import org.ossreviewtoolkit.utils.common.withoutPrefix

import org.semver4j.RangesList
import org.semver4j.RangesListFactory

/** Name of the scope with the regular dependencies. */
private const val DEPENDENCIES_SCOPE = "dependencies"

/** Name of the scope with optional dependencies. */
private const val OPTIONAL_DEPENDENCIES_SCOPE = "optionalDependencies"

/** Name of the scope with development dependencies. */
private const val DEV_DEPENDENCIES_SCOPE = "devDependencies"

/**
* The [Node package manager](https://www.npmjs.com/) for JavaScript.
*
Expand All @@ -93,12 +66,12 @@ private const val DEV_DEPENDENCIES_SCOPE = "devDependencies"
* information see the [documentation](https://docs.npmjs.com/cli/v8/commands/npm-install#strict-peer-deps) and the
* [NPM Blog](https://blog.npmjs.org/post/626173315965468672/npm-v7-series-beta-release-and-semver-major).
*/
open class Npm(
class Npm(
name: String,
analysisRoot: File,
analyzerConfig: AnalyzerConfiguration,
repoConfig: RepositoryConfiguration
) : PackageManager(name, analysisRoot, analyzerConfig, repoConfig), CommandLineTool {
) : Yarn(name, analysisRoot, analyzerConfig, repoConfig) {
companion object {
/** Name of the configuration option to toggle legacy peer dependency support. */
const val OPTION_LEGACY_PEER_DEPS = "legacyPeerDeps"
Expand All @@ -116,29 +89,9 @@ open class Npm(

private val legacyPeerDeps = options[OPTION_LEGACY_PEER_DEPS].toBoolean()

private val graphBuilder by lazy { DependencyGraphBuilder(NpmDependencyHandler(this)) }

private val npmViewCache = mutableMapOf<String, PackageJson>()

protected open fun hasLockfile(projectDir: File) = NodePackageManager.NPM.hasLockfile(projectDir)

/**
* Load the submodule directories of the project defined in [moduleDir].
*/
protected open fun loadWorkspaceSubmodules(moduleDir: File): Set<File> {
val nodeModulesDir = moduleDir.resolve("node_modules")
if (!nodeModulesDir.isDirectory) return emptySet()

val searchDirs = nodeModulesDir.walk().maxDepth(1).filter {
(it.isDirectory && it.name.startsWith("@")) || it == nodeModulesDir
}

return searchDirs.flatMapTo(mutableSetOf()) { dir ->
dir.walk().maxDepth(1).filter {
it.isDirectory && it.isSymbolicLink() && it != dir
}
}
}
override fun hasLockfile(projectDir: File) = NodePackageManager.NPM.hasLockfile(projectDir)

override fun command(workingDir: File?) = if (Os.isWindows) "npm.cmd" else "npm"

Expand All @@ -153,92 +106,7 @@ open class Npm(
checkVersion()
}

override fun createPackageManagerResult(projectResults: Map<File, List<ProjectAnalyzerResult>>) =
PackageManagerResult(projectResults, graphBuilder.build(), graphBuilder.packages())

override fun resolveDependencies(definitionFile: File, labels: Map<String, String>): List<ProjectAnalyzerResult> {
val workingDir = definitionFile.parentFile

return try {
stashDirectories(workingDir.resolve("node_modules")).use {
resolveDependenciesInternal(definitionFile)
}
} finally {
rawModuleInfoCache.clear()
}
}

// TODO: Add support for bundledDependencies.
private fun resolveDependenciesInternal(definitionFile: File): List<ProjectAnalyzerResult> {
val workingDir = definitionFile.parentFile

// Actually installing the dependencies is the easiest way to get the metadata of all transitive
// dependencies (i.e. their respective "package.json" files). As NPM uses a global cache, the same
// dependency is only ever downloaded once.
val installIssues = installDependencies(workingDir)

if (installIssues.any { it.severity == Severity.ERROR }) {
val project = runCatching {
parseProject(definitionFile, analysisRoot, managerName)
}.getOrElse {
logger.error { "Failed to parse project information: ${it.collectMessages()}" }
Project.EMPTY
}

return listOf(ProjectAnalyzerResult(project, emptySet(), installIssues))
}

val projectDirs = findWorkspaceSubmodules(workingDir).toSet() + definitionFile.parentFile

return projectDirs.map { projectDir ->
val issues = mutableListOf<Issue>().apply {
if (projectDir == workingDir) addAll(installIssues)
}

val project = runCatching {
parseProject(projectDir.resolve("package.json"), analysisRoot, managerName)
}.getOrElse {
issues += createAndLogIssue(
source = managerName,
message = "Failed to parse project information: ${it.collectMessages()}"
)

Project.EMPTY
}

val scopeNames = setOfNotNull(
// Optional dependencies are just like regular dependencies except that NPM ignores failures when
// installing them (see https://docs.npmjs.com/files/package.json#optionaldependencies), i.e. they are
// not a separate scope in ORT semantics.
buildDependencyGraphForScopes(
project,
projectDir,
setOf(DEPENDENCIES_SCOPE, OPTIONAL_DEPENDENCIES_SCOPE),
DEPENDENCIES_SCOPE,
projectDirs,
workspaceDir = workingDir
),

buildDependencyGraphForScopes(
project,
projectDir,
setOf(DEV_DEPENDENCIES_SCOPE),
DEV_DEPENDENCIES_SCOPE,
projectDirs,
workspaceDir = workingDir
)
)

ProjectAnalyzerResult(
project = project.copy(scopeNames = scopeNames),
// Packages are set later by createPackageManagerResult().
packages = emptySet(),
issues = issues
)
}
}

internal open fun getRemotePackageDetails(workingDir: File, packageName: String): PackageJson? {
override fun getRemotePackageDetails(workingDir: File, packageName: String): PackageJson? {
npmViewCache[packageName]?.let { return it }

return runCatching {
Expand All @@ -252,170 +120,7 @@ open class Npm(
}.getOrNull()
}

/** Cache for submodules identified by its moduleDir absolutePath */
private val submodulesCache = ConcurrentHashMap<String, Set<File>>()

/**
* Find the directories which are defined as submodules of the project within [moduleDir].
*/
private fun findWorkspaceSubmodules(moduleDir: File): Set<File> =
submodulesCache.getOrPut(moduleDir.absolutePath) {
loadWorkspaceSubmodules(moduleDir)
}

/**
* Retrieve all the dependencies of [project] from the given [scopes] and add them to the dependency graph under
* the given [targetScope]. Return the target scope name if dependencies are found; *null* otherwise.
*/
private fun buildDependencyGraphForScopes(
project: Project,
workingDir: File,
scopes: Set<String>,
targetScope: String,
projectDirs: Set<File>,
workspaceDir: File? = null
): String? {
if (excludes.isScopeExcluded(targetScope)) return null

val qualifiedScopeName = DependencyGraph.qualifyScope(project, targetScope)
val moduleInfo = checkNotNull(getModuleInfo(workingDir, scopes, projectDirs, listOfNotNull(workspaceDir)))

moduleInfo.dependencies.forEach { graphBuilder.addDependency(qualifiedScopeName, it) }

return targetScope.takeUnless { moduleInfo.dependencies.isEmpty() }
}

private fun getModuleInfo(
moduleDir: File,
scopes: Set<String>,
projectDirs: Set<File>,
ancestorModuleDirs: List<File> = emptyList(),
ancestorModuleIds: List<Identifier> = emptyList()
): NpmModuleInfo? {
val moduleInfo = parsePackageJson(moduleDir, scopes)
val dependencies = mutableSetOf<NpmModuleInfo>()
val packageType = managerName.takeIf { moduleDir.realFile() in projectDirs } ?: "NPM"

val moduleId = splitNpmNamespaceAndName(moduleInfo.name).let { (namespace, name) ->
Identifier(packageType, namespace, name, moduleInfo.version)
}

val cycleStartIndex = ancestorModuleIds.indexOf(moduleId)
if (cycleStartIndex >= 0) {
val cycle = (ancestorModuleIds.subList(cycleStartIndex, ancestorModuleIds.size) + moduleId)
.joinToString(" -> ")
logger.debug { "Not adding dependency '$moduleId' to avoid cycle: $cycle." }
return null
}

val pathToRoot = listOf(moduleDir) + ancestorModuleDirs
moduleInfo.dependencyNames.forEach { dependencyName ->
val dependencyModuleDirPath = findDependencyModuleDir(dependencyName, pathToRoot)

if (dependencyModuleDirPath.isNotEmpty()) {
val dependencyModuleDir = dependencyModuleDirPath.first()

getModuleInfo(
moduleDir = dependencyModuleDir,
scopes = setOf("dependencies", "optionalDependencies"),
projectDirs,
ancestorModuleDirs = dependencyModuleDirPath.subList(1, dependencyModuleDirPath.size),
ancestorModuleIds = ancestorModuleIds + moduleId
)?.let { dependencies += it }

return@forEach
}

logger.debug {
"It seems that the '$dependencyName' module was not installed as the package file could not be found " +
"anywhere in '${pathToRoot.joinToString()}'. This might be fine if the module is specific to a " +
"platform other than the one ORT is running on. A typical example is the 'fsevents' module."
}
}

return NpmModuleInfo(
id = moduleId,
workingDir = moduleDir,
packageFile = moduleInfo.packageJson,
dependencies = dependencies,
isProject = moduleDir.realFile() in projectDirs
)
}

/**
* An internally used data class with information about a module retrieved from the module's package.json. This
* information is further processed and eventually converted to an [NpmModuleInfo] object containing everything
* required by the Npm package manager.
*/
private data class RawModuleInfo(
val name: String,
val version: String,
val dependencyNames: Set<String>,
val packageJson: File
)

private val rawModuleInfoCache = mutableMapOf<Pair<File, Set<String>>, RawModuleInfo>()

private fun parsePackageJson(moduleDir: File, scopes: Set<String>): RawModuleInfo =
rawModuleInfoCache.getOrPut(moduleDir to scopes) {
val packageJsonFile = moduleDir.resolve("package.json")
logger.debug { "Parsing module info from '${packageJsonFile.absolutePath}'." }
val json = packageJsonFile.readTree()

val name = json["name"].textValueOrEmpty()
if (name.isBlank()) {
logger.warn {
"The '$packageJsonFile' does not set a name, which is only allowed for unpublished packages."
}
}

val version = json["version"].textValueOrEmpty()
if (version.isBlank()) {
logger.warn {
"The '$packageJsonFile' does not set a version, which is only allowed for unpublished packages."
}
}

val dependencyNames = scopes.flatMapTo(mutableSetOf()) { scope ->
// Yarn ignores "//" keys in the dependencies to allow comments, therefore ignore them here as well.
json[scope].fieldNamesOrEmpty().asSequence().filterNot { it == "//" }
}

RawModuleInfo(
name = name,
version = version,
dependencyNames = dependencyNames,
packageJson = packageJsonFile
)
}

/**
* Install dependencies using the given package manager command.
*/
private fun installDependencies(workingDir: File): List<Issue> {
requireLockfile(workingDir) { hasLockfile(workingDir) }

// Install all NPM dependencies to enable NPM to list dependencies.
val process = runInstall(workingDir)

val lines = process.stderr.lines()
val issues = mutableListOf<Issue>()

// Generally forward issues from the NPM CLI to the ORT NPM package manager. Lower the severity of warnings to
// hints, as warnings usually do not prevent the ORT NPM package manager from getting the dependencies right.
lines.groupLines("npm WARN ", "npm warn ").mapTo(issues) {
Issue(source = managerName, message = it, severity = Severity.HINT)
}

// For errors, however, something clearly went wrong, so keep the severity here.
lines.groupLines("npm ERR! ", "npm error ").mapTo(issues) {
Issue(source = managerName, message = it, severity = Severity.ERROR)
}

return issues
}

protected open fun runInstall(workingDir: File): ProcessCapture {
override fun runInstall(workingDir: File): ProcessCapture {
val options = listOfNotNull(
"--ignore-scripts",
"--no-audit",
Expand All @@ -427,18 +132,6 @@ open class Npm(
}
}

private fun findDependencyModuleDir(dependencyName: String, searchModuleDirs: List<File>): List<File> {
searchModuleDirs.forEachIndexed { index, moduleDir ->
// Note: resolve() also works for scoped dependencies, e.g. dependencyName = "@x/y"
val dependencyModuleDir = moduleDir.resolve("node_modules/$dependencyName")
if (dependencyModuleDir.isDirectory) {
return listOf(dependencyModuleDir) + searchModuleDirs.subList(index, searchModuleDirs.size)
}
}

return emptyList()
}

internal fun List<String>.groupLines(vararg markers: String): List<String> {
val ignorableLinePrefixes = setOf("code ", "errno ", "path ", "syscall ")
val singleLinePrefixes = setOf("deprecated ", "skipping integrity check for git dependency ")
Expand Down
Loading

0 comments on commit 743fd64

Please sign in to comment.