Refactor/common build utils

.fossa.yaml

@@ -16,7 +16,7 @@ analyze:
   #     type: folder
   modules:
     - name: fossa-cli
-      path: github.com/fossas/fossa-cli/cmd/fossa
+      path: ./cmd/fossa
       type: gopackage
     # - name: vendored-jquery
     #   path: vendor/jquery-stuff/bower.json

build/bower.go

@@ -1,13 +1,10 @@
 package build
 
 import (
-	"encoding/json"
 	"errors"
-	"io/ioutil"
+	"fmt"
 	"os"
-	"os/exec"
 	"path/filepath"
-	"strings"
 	"sync"
 
 	"github.com/bmatcuk/doublestar"
@@ -18,16 +15,15 @@ import (
 
 var bowerLogger = logging.MustGetLogger("bower")
 
-// BowerComponent implements Dependency for BowerBuilder.
+// BowerComponent implements Dependency for BowerBuilder
 type BowerComponent struct {
 	Name    string `json:"name"`
 	Version string `json:"version"`
 }
 
-// Fetcher always returns bower for BowerComponent. TODO: Support git and other
-// dependency sources.
+// Fetcher always returns bower for BowerComponent
 func (m BowerComponent) Fetcher() string {
-	return "bower" // TODO: support git and etc...
+	return "bower" // TODO: support `git` and etc...
 }
 
 // Package returns the package name for BowerComponent
@@ -40,8 +36,7 @@ func (m BowerComponent) Revision() string {
 	return m.Version
 }
 
-// BowerBuilder implements Builder for Bower.
-// These properties are public for the sake of serialization.
+// BowerBuilder implements Builder for Bower
 type BowerBuilder struct {
 	NodeCmd     string
 	NodeVersion string
@@ -50,117 +45,103 @@ type BowerBuilder struct {
 	BowerVersion string
 }
 
-// Initialize collects environment data for Bower builds
+// Initialize collects metadata on Node and Bower binaries
 func (builder *BowerBuilder) Initialize() error {
-	bowerLogger.Debugf("Initializing Bower builder...")
-	// Set Node context variables
-	nodeCmds := [3]string{os.Getenv("NODE_BINARY"), "node", "nodejs"}
-	for i := 0; true; i++ {
-		if i >= len(nodeCmds) {
-			return errors.New("could not find Nodejs binary (try setting $NODE_BINARY)")
-		}
-		if nodeCmds[i] == "" {
-			continue
-		}
+	bowerLogger.Debug("Initializing Bower builder...")
 
-		nodeVersionOutput, err := exec.Command(nodeCmds[i], "-v").Output()
-		if err == nil && nodeVersionOutput[0] == 'v' {
-			builder.NodeVersion = strings.TrimSpace(string(nodeVersionOutput))[1:]
-			builder.NodeCmd = nodeCmds[i]
-			break
-		}
+	// Set Node context variables
+	nodeCmd, nodeVersion, err := which("-v", os.Getenv("NODE_BINARY"), "node", "nodejs")
+	if err != nil {
+		return fmt.Errorf("could not find Node binary (try setting $NODE_BINARY): %s", err.Error())
 	}
+	builder.NodeCmd = nodeCmd
+	builder.NodeVersion = nodeVersion
 
 	// Set Bower context variables
-	builder.BowerCmd = os.Getenv("Bower_BINARY")
-	if builder.BowerCmd == "" {
-		builder.BowerCmd = "bower"
-	}
-
-	bowerVersionOutput, err := exec.Command(builder.BowerCmd, "-v").Output()
-	if err == nil && len(bowerVersionOutput) >= 5 {
-		builder.BowerVersion = strings.TrimSpace(string(bowerVersionOutput))
-	}
-
-	if builder.BowerCmd == "" || builder.BowerVersion == "" {
-		return errors.New("could not find Bower binary (try setting $BOWER_BINARY)")
+	bowerCmd, bowerVersion, err := which("-v", os.Getenv("BOWER_BINARY"), "bower")
+	if err != nil {
+		return fmt.Errorf("could not find Bower binary (try setting $BOWER_BINARY): %s", err.Error())
 	}
+	builder.BowerCmd = bowerCmd
+	builder.BowerVersion = bowerVersion
 
-	bowerLogger.Debugf("Initialized Bower builder: %#v", builder)
-
+	bowerLogger.Debugf("Done initializing Bower builder: %#v", builder)
 	return nil
 }
 
+// Build runs `bower install --production` and cleans with `rm -rf bower_components`
 func (builder *BowerBuilder) Build(m module.Module, force bool) error {
-	bowerLogger.Debugf("Running Bower build...")
+	bowerLogger.Debugf("Running Bower build: %#v", m, force)
+
 	if force {
-		bowerLogger.Debug("`force` flag is set; clearing `bower_components`...")
-		cmd := exec.Command("rm", "-rf", "bower_components")
-		cmd.Dir = m.Dir
-		_, err := cmd.Output()
+		_, _, err := runLogged(bowerLogger, m.Dir, "rm", "-rf", "bower_components")
 		if err != nil {
-			return err
+			return fmt.Errorf("could not remove Bower cache: %s", err.Error())
 		}
 	}
 
-	cmd := exec.Command(builder.BowerCmd, "install", "--production")
-	cmd.Dir = m.Dir
-	_, err := cmd.Output()
-	return err
+	_, _, err := runLogged(bowerLogger, m.Dir, builder.BowerCmd, "install", "--production")
+	if err != nil {
+		return fmt.Errorf("could not run Bower build: %s", err.Error())
+	}
+
+	bowerLogger.Debug("Done running Bower build.")
+	return nil
 }
 
-func (builder *BowerBuilder) Analyze(m module.Module, _ bool) ([]module.Dependency, error) {
-	bowerLogger.Debugf("Running analysis on Bower module...")
+// Analyze reads dependency manifests at `$PROJECT/**/bower_components/*/.bower.json`
+func (builder *BowerBuilder) Analyze(m module.Module, allowUnresolved bool) ([]module.Dependency, error) {
+	bowerLogger.Debugf("Running Bower analysis: %#v %#v", m, allowUnresolved)
+
+	// Find manifests.
 	bowerComponents, err := doublestar.Glob(filepath.Join(m.Dir, "**", "bower_components", "*", ".bower.json"))
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("could not find Bower dependency manifests: %s", err.Error())
 	}
-	bowerLogger.Debugf("Found %#v modules from globstar.", len(bowerComponents))
+	bowerLogger.Debugf("Found %#v modules from globstar: %#v", len(bowerComponents), bowerComponents)
 
+	// Read manifests.
 	var wg sync.WaitGroup
-	dependencies := make([]BowerComponent, len(bowerComponents))
+	deps := make([]module.Dependency, len(bowerComponents))
 	wg.Add(len(bowerComponents))
-
 	for i := 0; i < len(bowerComponents); i++ {
 		go func(modulePath string, index int, wg *sync.WaitGroup) {
 			defer wg.Done()
 
-			dependencyManifest, err := ioutil.ReadFile(modulePath)
-			if err != nil {
-				bowerLogger.Warningf("Error parsing Module: %#v", modulePath)
-				return
-			}
+			var bowerComponent BowerComponent
+			parseLogged(bowerLogger, modulePath, &bowerComponent)
 
 			// Write directly to a reserved index for thread safety
-			json.Unmarshal(dependencyManifest, &dependencies[index])
+			deps[index] = bowerComponent
 		}(bowerComponents[i], i, &wg)
 	}
-
 	wg.Wait()
 
-	var deps []module.Dependency
-	for i := 0; i < len(dependencies); i++ {
-		deps = append(deps, dependencies[i])
-	}
-
+	bowerLogger.Debugf("Done running Bower analysis: %#v", deps)
 	return deps, nil
 }
 
-func (builder *BowerBuilder) IsBuilt(m module.Module, _ bool) (bool, error) {
-	bowerComponentsPath := filepath.Join(m.Dir, "bower_components")
-	bowerLogger.Debugf("Checking bower_components at %#v", bowerComponentsPath)
+// IsBuilt checks for the existence of `$PROJECT/bower_components`
+func (builder *BowerBuilder) IsBuilt(m module.Module, allowUnresolved bool) (bool, error) {
+	bowerLogger.Debug("Checking Bower build: %#v %#v", m, allowUnresolved)
+
 	// TODO: Check if the installed modules are consistent with what's in the
 	// actual manifest.
-	if _, err := os.Stat(bowerComponentsPath); err == nil {
-		return true, nil
+	isBuilt, err := hasFile(m.Dir, "bower_components")
+	if err != nil {
+		return false, fmt.Errorf("could not find Bower dependencies folder: %s", err.Error())
 	}
-	return false, nil
+
+	bowerLogger.Debugf("Done checking Bower build: %#v", isBuilt)
+	return isBuilt, nil
 }
 
+// IsModule is not implemented
 func (builder *BowerBuilder) IsModule(target string) (bool, error) {
 	return false, errors.New("IsModule is not implemented for BowerBuilder")
 }
 
+// InferModule is not implemented
 func (builder *BowerBuilder) InferModule(target string) (module.Module, error) {
 	return module.Module{}, errors.New("InferModule is not implemented for BowerBuilder")
 }

build/common.go

@@ -0,0 +1,143 @@
+package build
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	logging "github.com/op/go-logging"
+)
+
+var commonLogger = logging.MustGetLogger("common")
+
+// Utilities for finding files and manifests
+func hasFile(elem ...string) (bool, error) {
+	_, err := os.Stat(filepath.Join(elem...))
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return !os.IsNotExist(err), err
+}
+
+func orPredicates(predicates ...fileChecker) fileChecker {
+	return func(path string) (bool, error) {
+		for _, predicate := range predicates {
+			ok, err := predicate(path)
+			if err != nil {
+				return false, err
+			}
+			if ok {
+				return ok, nil
+			}
+		}
+		return false, nil
+	}
+}
+
+type fileChecker func(path string) (bool, error)
+
+func findAncestor(stopWhen fileChecker, path string) (string, bool, error) {
+	absPath, err := filepath.Abs(path)
+	if absPath == string(filepath.Separator) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	stop, err := stopWhen(absPath)
+	if err != nil {
+		return "", false, err
+	}
+	if stop {
+		return absPath, true, nil
+	}
+	return findAncestor(stopWhen, filepath.Dir(path))
+}
+
+// Utilities around `exec.Command`
+func run(name string, arg ...string) (string, string, error) {
+	var stderr bytes.Buffer
+	cmd := exec.Command(name, arg...)
+	cmd.Stderr = &stderr
+	stdout, err := cmd.Output()
+	return string(stdout), stderr.String(), err
+}
+
+func runInDir(dir string, name string, arg ...string) (string, string, error) {
+	var stderr bytes.Buffer
+	cmd := exec.Command(name, arg...)
+	cmd.Dir = dir
+	cmd.Stderr = &stderr
+	stdout, err := cmd.Output()
+	return string(stdout), stderr.String(), err
+}
+
+// Utilities for debug logging
+func runLogged(logger *logging.Logger, dir string, name string, arg ...string) (string, string, error) {
+	cmd := strings.Join(append([]string{name}, arg...), " ")
+	logger.Debugf("Running `%s`...", cmd)
+	stdout, stderr, err := runInDir(dir, name, arg...)
+	if err != nil {
+		logger.Debugf("Running `%s` failed: %#v %#v", cmd, err, stderr)
+		return "", "", fmt.Errorf("running `%s` failed: %#v %#v", cmd, err, stderr)
+	}
+	logger.Debugf("Done running `%s`: %#v %#v", stdout, stderr)
+	return stdout, stderr, nil
+}
+
+func parseLogged(logger *logging.Logger, file string, v interface{}) error {
+	return parseLoggedWithUnmarshaller(logger, file, v, json.Unmarshal)
+}
+
+type unmarshaller func(data []byte, v interface{}) error
+
+func parseLoggedWithUnmarshaller(logger *logging.Logger, file string, v interface{}, unmarshal unmarshaller) error {
+	logger.Debugf("Parsing %s...", file)
+
+	contents, err := ioutil.ReadFile(file)
+	if err != nil {
+		logger.Debugf("Error reading %s: %s", file, err.Error())
+		return err
+	}
+	err = unmarshal(contents, v)
+	if err != nil {
+		logger.Debugf("Error parsing %s: %#v %#v", file, err, contents)
+		return err
+	}
+
+	logger.Debugf("Done parsing %s.", file)
+	return nil
+}
+
+// Utilities for detecting which binary to use
+type versionResolver func(cmd string) (string, error)
+
+func whichWithResolver(cmds []string, getVersion versionResolver) (string, string, error) {
+	for _, cmd := range cmds {
+		version, err := getVersion(cmd)
+		if err == nil {
+			return cmd, version, nil
+		}
+		commonLogger.Debugf("Tried resolving `%s` but did not work: %#v %#v", cmd, err, version)
+	}
+	return "", "", errors.New("could not resolve version")
+}
+
+func which(versionFlags string, cmds ...string) (string, string, error) {
+	return whichWithResolver(cmds, func(cmd string) (string, error) {
+		stdout, stderr, err := run(cmd, strings.Split(versionFlags, " ")...)
+		if err != nil {
+			return "", err
+		}
+		if stdout == "" {
+			return stderr, nil
+		}
+		return stdout, nil
+	})
+}