feat(builders): Add Pip support

.vscode/settings.json

@@ -0,0 +1,9 @@
+{
+  "files.watcherExclude": {
+    "**/.git/objects/**": true,
+    "**/.git/subtree-cache/**": true,
+    "**/node_modules/**": true,
+    "text/fixtures/*/**": true,
+    "vendor/**": true
+  }
+}

builders/builder.go

@@ -37,6 +37,8 @@ func New(moduleType module.Type) Builder {
 		return &MavenBuilder{}
 	case module.Nodejs:
 		return &NodeJSBuilder{}
+	case module.Pip:
+		return &PipBuilder{}
 	case module.Ruby:
 		return &RubyBuilder{}
 	case module.SBT:

builders/pip.go

@@ -0,0 +1,164 @@
+package builders
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/fossas/fossa-cli/module"
+	logging "github.com/op/go-logging"
+)
+
+var pipLogger = logging.MustGetLogger("pip")
+
+// PythonPackage implements Dependency for PipBuilder.
+type PythonPackage struct {
+	Name    string `json:"name"`
+	Version string `json:"version"`
+}
+
+// Fetcher always returns pip for PythonPackage. TODO: Support git and other
+// dependency sources.
+func (m PythonPackage) Fetcher() string {
+	return "pip" // TODO: support git and etc...
+}
+
+// Package returns the package name for PythonPackage
+func (m PythonPackage) Package() string {
+	return m.Name
+}
+
+// Revision returns the version for PythonPackage
+func (m PythonPackage) Revision() string {
+	return m.Version
+}
+
+// PipBuilder implements Builder for Pip.
+// These properties are public for the sake of serialization.
+type PipBuilder struct {
+	PythonCmd     string
+	PythonVersion string
+
+	PipCmd     string
+	PipVersion string
+}
+
+// Initialize collects metadata on Python and Pip binaries
+func (builder *PipBuilder) Initialize() error {
+	pipLogger.Debug("Initializing Pip builder...")
+
+	// Set Python context variables
+	pythonCmd, pythonVersion, err := which("--version", os.Getenv("PYTHON_BINARY"), "python", "python3", "python27", "python2")
+	if err != nil {
+		pipLogger.Warningf("Could not find Python binary (try setting $PYTHON_BINARY): %s", err.Error())
+	}
+	builder.PythonCmd = pythonCmd
+	builder.PythonVersion = pythonVersion
+
+	// Set Pip context variables
+	pipCmd, pipVersion, pipErr := which("--version", os.Getenv("PIP_BINARY"), "pip")
+	builder.PipCmd = pipCmd
+	builder.PipVersion = pipVersion
+	if pipErr != nil {
+		pipLogger.Warningf("No supported Python build tools detected (try setting $PIP_BINARY): %#v", pipErr)
+	}
+
+	pipLogger.Debugf("Initialized Pip builder: %#v", builder)
+	return nil
+}
+
+// Build runs `pip install -r requirements.txt`
+func (builder *PipBuilder) Build(m module.Module, force bool) error {
+	pipLogger.Debugf("Running Pip build: %#v %#v", m, force)
+
+	_, _, err := runLogged(pipLogger, m.Dir, builder.PipCmd, "install", "-r", "requirements.txt")
+	if err != nil {
+		return fmt.Errorf("could not run Pip build: %s", err.Error())
+	}
+
+	pipLogger.Debug("Done running Pip build.")
+	return nil
+}
+
+// Analyze reads `requirements.txt`
+func (builder *PipBuilder) Analyze(m module.Module, allowUnresolved bool) ([]module.Dependency, error) {
+	pipLogger.Debugf("Running Pip analysis: %#v %#v", m, allowUnresolved)
+
+	lockfile := filepath.Join(m.Dir, "requirements.txt")
+	lockfileContents, err := ioutil.ReadFile(lockfile)
+	if err != nil {
+		goLogger.Debugf("Error reading %s: %s", lockfile, err.Error())
+		return nil, fmt.Errorf("could not read %s: %s", lockfile, err.Error())
+	}
+
+	lines := strings.Split(string(lockfileContents), "\n")
+	lockfileVersions := make(map[string]string)
+	for _, line := range lines {
+		trimmedLine := strings.TrimSpace(line)
+		if len(trimmedLine) > 0 && trimmedLine[0] != '#' {
+			// TODO: support other kinds of constraints
+			sections := strings.Split(trimmedLine, "==")
+			lockfileVersions[sections[0]] = sections[1]
+		}
+	}
+
+	var deps []module.Dependency
+	for pkg, version := range lockfileVersions {
+		deps = append(deps, PythonPackage{Name: pkg, Version: version})
+	}
+
+	pipLogger.Debugf("Done running Pip analysis: %#v", deps)
+	return deps, nil
+}
+
+// IsBuilt checks for the existence of `requirements.txt`
+func (builder *PipBuilder) IsBuilt(m module.Module, allowUnresolved bool) (bool, error) {
+	pipLogger.Debugf("Checking Pip build: %#v %#v", m, allowUnresolved)
+
+	// TODO: support `pip freeze` output and other alternative methods?
+	isBuilt, err := hasFile(m.Dir, "requirements.txt")
+	if err != nil {
+		return false, fmt.Errorf("could not find `requirements.txt`: %s", err.Error())
+	}
+
+	pipLogger.Debugf("Done checking Pip build: %#v", isBuilt)
+	return isBuilt, nil
+}
+
+// IsModule is not implemented
+func (builder *PipBuilder) IsModule(target string) (bool, error) {
+	return false, errors.New("IsModule is not implemented for PipBuilder")
+}
+
+// DiscoverModules builds ModuleConfigs for any `requirements.txt` files
+func (builder *PipBuilder) DiscoverModules(dir string) ([]module.Config, error) {
+	var moduleConfigs []module.Config
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			pipLogger.Debugf("Failed to access path %s: %s\n", path, err.Error())
+			return err
+		}
+
+		if !info.IsDir() && info.Name() == "requirements.txt" {
+			moduleName := filepath.Base(filepath.Dir(path))
+
+			pipLogger.Debugf("Found Python package: %s (%s)", path, moduleName)
+			path, _ = filepath.Rel(dir, path)
+			moduleConfigs = append(moduleConfigs, module.Config{
+				Name: moduleName,
+				Path: path,
+				Type: string(module.Pip),
+			})
+		}
+		return nil
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("Could not find Python package manifests: %s", err.Error())
+	}
+
+	return moduleConfigs, nil
+}

module/module.go

@@ -61,6 +61,9 @@ func New(moduleType Type, conf Config) (Module, error) {
 	case Nodejs:
 		manifestName = "package.json"
 		break
+	case Pip:
+		manifestName = "requirements.txt"
+		break
 	case Ruby:
 		manifestName = "Gemfile"
 		break

module/types.go

@@ -27,6 +27,8 @@ const (
 	SBT = Type("sbt")
 	// Gradle is the module type for gradle.org
 	Gradle = Type("gradle")
+	// Pip is the module type for https://pip.pypa.io/en/stable/
+	Pip = Type("pip")
 
 	// Ecosystems where many tools behave similarly
 
@@ -43,7 +45,7 @@ const (
 )
 
 // Types holds the list of all available module types for analysis
-var Types = []Type{Bower, Composer, Maven, SBT, Gradle, Ruby, Nodejs, Golang, VendoredArchives}
+var Types = []Type{Bower, Composer, Maven, SBT, Gradle, Ruby, Nodejs, Golang, VendoredArchives, Pip}
 
 // Parse returns a module Type given a string
 func Parse(key string) (Type, error) {
@@ -76,6 +78,14 @@ func Parse(key string) (Type, error) {
 	case "mvn":
 		return Maven, nil
 
+	// Python aliases:
+	case "python":
+		fallthrough
+	case "py":
+		fallthrough
+	case "pip":
+		return Pip, nil
+
 	// Ruby aliases
 	case "bundler":
 		fallthrough

test/fixtures/pip/requirements.txt

@@ -0,0 +1,45 @@
+amqp==2.2.2
+asn1crypto==0.24.0
+attrs==17.4.0
+Automat==0.6.0
+billiard==3.5.0.3
+celery==4.1.0
+certifi==2018.1.18
+cffi==1.11.5
+chardet==3.0.4
+constantly==15.1.0
+cryptography==2.2.2
+cssselect==1.0.3
+cycler==0.10.0
+Django==2.0.3
+hyperlink==18.0.0
+idna==2.6
+incremental==17.5.0
+kiwisolver==1.0.1
+kombu==4.1.0
+lxml==4.2.1
+matplotlib==2.2.2
+nose==1.3.7
+numpy==1.14.2
+parsel==1.4.0
+Pillow==5.0.0
+pyasn1==0.4.2
+pyasn1-modules==0.2.1
+pycparser==2.18
+PyDispatcher==2.0.5
+pyOpenSSL==17.5.0
+pyparsing==2.2.0
+python-dateutil==2.7.2
+pytz==2018.3
+queuelib==1.5.0
+requests==2.18.4
+scapy==2.4.0
+Scrapy==1.5.0
+service-identity==17.0.0
+six==1.11.0
+SQLAlchemy==1.2.5
+Twisted==17.9.0
+urllib3==1.22
+vine==1.1.4
+w3lib==1.19.0
+zope.interface==4.4.3