Skip to content

Commit

Permalink
Pip: Replace pipdeptree with python-inspector
Browse files Browse the repository at this point in the history 
This PR replaces pipdeptree with python-inspector to resolve
Python packages dependencies found in requirement files.
python-inspector can resolve dependencies for any target
Python version and OS (and not only the one running the tool).
In this integration in ORT, it replaces pipdeptree pretty much
in place as python-inspector implements a similar output data
structure by design to ease the integration.

Reference: https://github.com/nexB/python-inspector
Reference: oss-review-toolkit#4637
Reference: oss-review-toolkit#3671
Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>
Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>
  • Loading branch information
@TG1999
TG1999 committed Aug 23, 2022
1 parent 65c1565 commit d97b557
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 42 deletions.
6 changes: 6 additions & 0 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ ARG CRT_FILES=""
# Set this to the ScanCode version to use.
ARG SCANCODE_VERSION="30.1.0"

# Set this to the Python Inspector version to use.
ARG PYTHON_INSPECTOR_VERSION="0.6.4"

FROM eclipse-temurin:11-jdk-jammy AS build

COPY . /usr/local/src/ort
Expand Down Expand Up @@ -156,6 +159,9 @@ RUN /opt/ort/bin/import_proxy_certs.sh && \
ARG SCANCODE_VERSION
RUN pip install --no-cache-dir scancode-toolkit==$SCANCODE_VERSION

ARG PYTHON_INSPECTOR_VERSION
RUN pip install --no-cache-dir python-inspector==$PYTHON_INSPECTOR_VERSION

FROM run AS dist

ARG ORT_VERSION
Expand Down
102 changes: 60 additions & 42 deletions analyzer/src/main/kotlin/managers/Pip.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ import org.apache.logging.log4j.kotlin.Logging
import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory
import org.ossreviewtoolkit.analyzer.PackageManager
import org.ossreviewtoolkit.downloader.VersionControlSystem
import org.ossreviewtoolkit.model.EMPTY_JSON_NODE
import org.ossreviewtoolkit.model.Hash
import org.ossreviewtoolkit.model.Identifier
import org.ossreviewtoolkit.model.Package
Expand Down Expand Up @@ -62,14 +61,8 @@ import org.ossreviewtoolkit.utils.spdx.SpdxLicenseIdExpression
// https://pip.pypa.io/en/stable/news/#id176.
private const val PIP_VERSION = "20.3.4"

// See https://github.com/naiquevin/pipdeptree.
private const val PIPDEPTREE_VERSION = "2.2.1"

private val PHONY_DEPENDENCIES = mapOf(
"pipdeptree" to "", // A dependency of pipdeptree itself.
"pkg-resources" to "0.0.0", // Added by a bug with some Ubuntu distributions.
"setuptools" to "", // A dependency of pipdeptree itself.
"wheel" to "" // A dependency of pipdeptree itself.
)

private fun isPhonyDependency(name: String, version: String): Boolean =
Expand Down Expand Up @@ -142,6 +135,34 @@ object PythonVersion : CommandLineTool, Logging {
}
}

object PythonInspector : CommandLineTool {
override fun command(workingDir: File?) = "python-inspector"

fun runPythonInspector(
workingDir: File,
outputFile: String,
requirementsFile: String?,
setupPyFile: String?,
pythonVersion: String = "38",
): ProcessCapture {
var commandLineOptions = listOf(
"python-inspector",
"--python-version", pythonVersion,
"--json-pdt", outputFile,
)

if (requirementsFile != null) {
commandLineOptions += listOf("--requirement", requirementsFile)
} else if (setupPyFile != null) {
commandLineOptions += listOf("--setup-py", setupPyFile)
}

val process = ProcessCapture(workingDir, *commandLineOptions.toTypedArray())
process.requireSuccess()
return process
}
}

/**
* The [PIP](https://pip.pypa.io/) package manager for Python. Also see
* [install_requires vs requirements files](https://packaging.python.org/discussions/install-requires-vs-requirements/)
Expand Down Expand Up @@ -208,16 +229,15 @@ class Pip(

override fun resolveDependencies(definitionFile: File, labels: Map<String, String>): List<ProjectAnalyzerResult> {
// For an overview, dependency resolution involves the following steps:
// 1. Install dependencies via pip (inside a virtualenv, for isolation from globally installed packages).
// 2. Get metadata about the local project via `python setup.py`.
// 3. Get the hierarchy of dependencies via pipdeptree.
// 4. Get additional remote package metadata via PyPIJSON.
// 1. Get metadata about the local project via `python setup.py`.
// 2. Get the hierarchy of dependencies via python-inspector.
// 3. Get additional remote package metadata via PyPI JSON.

val workingDir = definitionFile.parentFile
val virtualEnvDir = setupVirtualEnv(workingDir, definitionFile)

val project = getProjectBasics(definitionFile, virtualEnvDir)
val (packages, installDependencies) = getInstallDependencies(definitionFile, virtualEnvDir, project.id.name)
val (packages, installDependencies) = getInstallDependencies(definitionFile, virtualEnvDir)

// TODO: Handle "extras" and "tests" dependencies.
val scopes = sortedSetOf(
Expand Down Expand Up @@ -312,37 +332,43 @@ class Pip(
}

private fun getInstallDependencies(
definitionFile: File, virtualEnvDir: File, projectName: String
definitionFile: File, virtualEnvDir: File,
): Pair<SortedSet<Package>, SortedSet<PackageReference>> {
val packages = sortedSetOf<Package>()
val installDependencies = sortedSetOf<PackageReference>()

val workingDir = definitionFile.parentFile

// List all packages installed locally in the virtualenv.
val pipdeptree = runInVirtualEnv(virtualEnvDir, workingDir, "pipdeptree", "-l", "--json-tree")
val jsonFile = createOrtTempDir().resolve("python-inspector.json")

val pythonInspector = if (definitionFile.name == "setup.py") {
PythonInspector.runPythonInspector(
workingDir = workingDir,
outputFile = jsonFile.absolutePath,
setupPyFile = definitionFile.absolutePath,
requirementsFile = null
)
} else {
PythonInspector.runPythonInspector(
workingDir = workingDir,
outputFile = jsonFile.absolutePath,
requirementsFile = definitionFile.absolutePath,
setupPyFile = null
)
}

// Get the locally available metadata for all installed packages as a fallback.
val installedPackages = getInstalledPackagesWithLocalMetaData(virtualEnvDir, workingDir).associateBy { it.id }

if (pipdeptree.isSuccess) {
val fullDependencyTree = jsonMapper.readTree(pipdeptree.stdout)

val projectDependencies = if (definitionFile.name == "setup.py") {
// The tree contains a root node for the project itself and pipdeptree's dependencies are also at the
// root next to it, as siblings.
fullDependencyTree.find {
it["package_name"].textValue() == projectName
}?.get("dependencies") ?: run {
logger.info { "The '$projectName' project does not declare any dependencies." }
EMPTY_JSON_NODE
}
} else {
// The tree does not contain a node for the project itself. Its dependencies are on the root level
// together with the dependencies of pipdeptree itself, which we need to filter out.
fullDependencyTree.filterNot {
isPhonyDependency(it["package_name"].textValue(), it["installed_version"].textValueOrEmpty())
}
if (pythonInspector.isSuccess) {
val fullDependencyTree = jsonMapper.readTree(jsonFile)
jsonFile.parentFile.safeDeleteRecursively(force = true)

val projectDependencies = fullDependencyTree.filterNot {
isPhonyDependency(
it["package_name"].textValue(),
it["installed_version"].textValueOrEmpty()
)
}

val allIds = sortedSetOf<Identifier>()
Expand All @@ -355,7 +381,7 @@ class Pip(
}
} else {
logger.error {
"Unable to determine dependencies for project in directory '$workingDir':\n${pipdeptree.stderr}"
"Unable to determine dependencies for project in directory '$workingDir':\n${pythonInspector.stderr}"
}
}

Expand Down Expand Up @@ -503,14 +529,6 @@ class Pip(
}
pip.requireSuccess()

// Install pipdeptree inside the virtualenv as that's the only way to make it report only the project's
// dependencies instead of those of all (globally) installed packages, see
// https://github.com/naiquevin/pipdeptree#known-issues.
// We only depend on pipdeptree to be at least version 0.5.0 for JSON output, but we stick to a fixed
// version to be sure to get consistent results.
pip = runPipInVirtualEnv(virtualEnvDir, workingDir, "install", "pipdeptree==$PIPDEPTREE_VERSION")
pip.requireSuccess()

// TODO: Find a way to make installation of packages with native extensions work on Windows where often the
// appropriate compiler is missing / not set up, e.g. by using pre-built packages from
// http://www.lfd.uci.edu/~gohlke/pythonlibs/
Expand Down

0 comments on commit d97b557

Please sign in to comment.