Skip to content

Commit

Permalink
Merge branch 'master' into alex/managed-builds
Browse files Browse the repository at this point in the history
  • Loading branch information
xizhao authored Mar 2, 2018
2 parents c3ccc74 + ddcc341 commit 3105635
Show file tree
Hide file tree
Showing 17 changed files with 258 additions and 32 deletions.
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,6 @@
[submodule "test/fixtures/sbt"]
path = test/fixtures/sbt
url = https://github.com/apache/spark
[submodule "test/fixtures/gradle"]
path = test/fixtures/gradle
url = https://github.com/google/iosched
5 changes: 2 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

**Features:**

- Supports over 15+ build systems (npm, golang, maven, etc...)
- Supports over 15+ build systems (npm, golang, maven, gradle, etc...)
- Supports monoliths; auto-detects and configures for multiple builds and modules in one codebase
- Fast and portable; a single cross-platform binary you can drop into CI builds and test your dependencies live
- Integrates with https://fossa.io for metadata and license auditing across open source packages
Expand Down Expand Up @@ -92,9 +92,8 @@ Instead of trying to guess at your build system's behavior, `fossa` runs locally
`fossa` supports a wide variety of languages, package managers, and build tools out of the box:

- JavaScript: `bower`, `npm`, `yarn`
- Java: `mvn`
- Java/Scala: `mvn`, `gradle`, `sbt`
- Ruby: `bundler`
- Scala: `sbt`
- PHP: `composer`
- Go: `dep`, `glide`, `godep`, `govendor`, `vndr`, `gdm`
- Archives: `*.rpm`
Expand Down
2 changes: 2 additions & 0 deletions builders/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ func New(moduleType config.ModuleType) Builder {
return &ComposerBuilder{}
case config.Golang:
return &GoBuilder{}
case config.Gradle:
return &GradleBuilder{}
case config.Maven:
return &MavenBuilder{}
case config.Nodejs:
Expand Down
149 changes: 149 additions & 0 deletions builders/gradle.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
package builders

import (
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"

logging "github.com/op/go-logging"

"github.com/fossas/fossa-cli/config"
"github.com/fossas/fossa-cli/module"
)

var gradleLogger = logging.MustGetLogger("gradle")

// GradleBuilder implements Builder for build.gradle builds
type GradleBuilder struct {
GradleCmd string
GradleVersion string
}

// Initialize collects metadata on Java and Maven binaries
func (builder *GradleBuilder) Initialize() error {
gradleLogger.Debug("Initializing Gradle builder...")

// Set Gradle context variables
gradleCmd, gradleVersionOut, err := which("--version --offline", os.Getenv("GRADLE_BINARY"), "./gradlew", "gradle")
if err == nil {
builder.GradleCmd = gradleCmd

gradleVersionMatchRe := regexp.MustCompile(`Gradle ([0-9]+\.[0-9]+.\w+)`)
match := gradleVersionMatchRe.FindStringSubmatch(gradleVersionOut)
if len(match) == 2 {
builder.GradleVersion = match[1]
}
}

if builder.GradleCmd == "" {
return fmt.Errorf("could not find Gradle binary or wrapper (try setting $GRADLE_BINARY)")
}

gradleLogger.Debugf("Done initializing Gradle builder: %#v", builder)
return nil
}

// Build is not implemented
func (builder *GradleBuilder) Build(m module.Module, force bool) error {
return nil
}

// Analyze parses the output of `gradle -q app:dependencies`
func (builder *GradleBuilder) Analyze(m module.Module, allowUnresolved bool) ([]module.Dependency, error) {
gradleLogger.Debugf("Running Gradle analysis: %#v %#v in %s", m, allowUnresolved, m.Dir)

// TODO: We need to let the user configure the right configurations
// NOTE: we are intentionally using exec.Command over runLogged here, due to path issues with defining cmd.Dir
dependenciesOutput, err := exec.Command(builder.GradleCmd, m.Name+":dependencies", "-q", "--configuration=compile", "--offline", "-a").Output()
if len(dependenciesOutput) == 0 || err != nil {
return nil, fmt.Errorf("could not run Gradle task %s:dependencies", m.Name)
}

var deps []module.Dependency
// Parse out any line that matches - (groupId):(artifactId):(version) -> (mediatedVersion) where `-> (mediatedVersion)`` is optional
dependenciesRe := regexp.MustCompile(`- ([\w\.-]+):([\w\.-]+):([\w\.-]+)( -> ([\w\.-]+))?`)
for _, line := range strings.Split(string(dependenciesOutput), "\n") {
trimmed := strings.TrimSpace(line)
if len(trimmed) > 0 {
parsedDependencyLine := dependenciesRe.FindStringSubmatch(trimmed)
if len(parsedDependencyLine) >= 4 {
groupID := parsedDependencyLine[1]
artifactID := parsedDependencyLine[2]
revisionID := parsedDependencyLine[3]
if len(parsedDependencyLine) == 6 && parsedDependencyLine[5] != "" {
revisionID = parsedDependencyLine[5]
}
gradleLogger.Debugf("Discovered maven artifact (%s, %s, %s)", trimmed, groupID, artifactID, revisionID)
deps = append(deps, MavenArtifact{
Name: fmt.Sprintf("%s:%s", groupID, artifactID),
Version: revisionID,
})
}
}
}

gradleLogger.Debugf("Done running Gradle analysis: %#v", deps)
return deps, nil
}

// IsBuilt is not implemented
func (builder *GradleBuilder) IsBuilt(m module.Module, allowUnresolved bool) (bool, error) {
return true, nil
}

// IsModule is not implemented
func (builder *GradleBuilder) IsModule(target string) (bool, error) {
return false, errors.New("IsModule is not implemented for GradleBuilder")
}

// DiscoverModules finds either a root build.gradle file in the specified dir
func (builder *GradleBuilder) DiscoverModules(dir string) ([]config.ModuleConfig, error) {
if err := builder.Initialize(); err != nil {
return nil, err
}
// Look for the root Gradle build
// TODO: support custom *.gradle files
if _, err := os.Stat(filepath.Join(dir, "build.gradle")); err == nil {
// Use bare exec.Command as runLogged errors when resolving outside of dir
taskListOutput, gradleTaskErr := exec.Command(builder.GradleCmd, "tasks", "--all", "-q", "-a", "--offline").Output()
if len(taskListOutput) > 0 && gradleTaskErr == nil {
// Search for subprojects using Gradle task list instead of grepping for build.gradle
var moduleConfigurations []config.ModuleConfig
// NOTE: this leaves out the root ("") dependencies task. To include, replace with `(\w+:)?dependencies -`
taskListRe := regexp.MustCompile(`\w+:dependencies -`)
for _, line := range strings.Split(string(taskListOutput), "\n") {
trimmed := strings.TrimSpace(line)
if len(trimmed) > 0 {
depMatchIndices := taskListRe.FindStringIndex(trimmed)
if len(depMatchIndices) > 0 && depMatchIndices[0] == 0 {
gradleTask := strings.Split(trimmed, " - ")[0]
gradleLogger.Debugf("found gradle dependencies task: %s", gradleTask)
moduleConfigurations = append(moduleConfigurations, config.ModuleConfig{
Name: strings.Split(gradleTask, ":")[0], // Name is the gradle task name
Path: "build.gradle",
Type: "gradle",
})
}
}
}

return moduleConfigurations, nil
}

// Fall back to "app" as default task, even though technically it would be "" (root)
return []config.ModuleConfig{
config.ModuleConfig{
Name: "app",
Path: "build.gradle",
Type: "gradle",
},
}, nil
}

// TODO: support multiple build.gradle files
return nil, nil
}
2 changes: 1 addition & 1 deletion builders/maven.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ func (builder *MavenBuilder) DiscoverModules(dir string) ([]config.ModuleConfig,
_, err := os.Stat(filepath.Join(dir, "pom.xml"))
if err == nil {
// Root pom found; parse and return
artifactName := filepath.Dir(dir)
artifactName := filepath.Base(filepath.Dir(dir))
var rootPom POMFile
if err := parseLoggedWithUnmarshaller(mavenLogger, filepath.Join(dir, "pom.xml"), &rootPom, xml.Unmarshal); err == nil {
if rootPom.Name != "" {
Expand Down
1 change: 1 addition & 0 deletions builders/nodejs.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ func (builder *NodeJSBuilder) DiscoverModules(dir string) ([]config.ModuleConfig
}

nodejsLogger.Debugf("Found NodeJS package: %s (%s)", path, moduleName)
path, _ = filepath.Rel(dir, path)
moduleConfigs = append(moduleConfigs, config.ModuleConfig{
Name: moduleName,
Path: path,
Expand Down
3 changes: 2 additions & 1 deletion builders/ruby.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ func (builder *RubyBuilder) DiscoverModules(dir string) ([]config.ModuleConfig,
}
moduleConfigs := make([]config.ModuleConfig, len(gemFilePaths))
for i, path := range gemFilePaths {
gemName := filepath.Dir(path)
gemName := filepath.Base(filepath.Dir(path))
// infer title from *.gemspec in directory if exists
gemSpecs, err := doublestar.Glob(filepath.Join(filepath.Dir(path), "*.gemspec"))
if err == nil && len(gemSpecs) > 0 {
Expand All @@ -175,6 +175,7 @@ func (builder *RubyBuilder) DiscoverModules(dir string) ([]config.ModuleConfig,
}
}
}
path, _ = filepath.Rel(dir, path)
moduleConfigs[i] = config.ModuleConfig{
Name: gemName,
Path: path,
Expand Down
5 changes: 4 additions & 1 deletion cmd/fossa/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func doInit(conf *config.CLIConfig, overwrite bool, includeAll bool) error {
// Filter suspicious modules
var filteredModuleConfigs []config.ModuleConfig
for _, c := range conf.Modules {
if matched, err := regexp.MatchString("(docs?/|test|example|vendor/)", c.Path); err != nil || matched != true {
if matched, err := regexp.MatchString("(docs?/|test|example|vendor/|node_modules/|.srclib-cache/|spec/|Godeps/|.git/|bower_components/)", c.Path); err != nil || matched != true {
filteredModuleConfigs = append(filteredModuleConfigs, c)
} else {
initLogger.Warningf("Filtering out suspicious module: %s (%s)", c.Name, c.Path)
Expand All @@ -70,6 +70,9 @@ func findModules(dir string) ([]config.ModuleConfig, error) {
var moduleConfigs []config.ModuleConfig
for _, t := range config.ModuleTypes {
builder := builders.New(t)
if builder == nil {
initLogger.Warningf("No builder available for module type: %s", t)
}
foundModules, err := builder.DiscoverModules(dir)
if err != nil {
lastError = err
Expand Down
7 changes: 6 additions & 1 deletion cmd/fossa/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ func main() {
cli.StringFlag{Name: "p, project", Usage: projectUsage},
cli.StringFlag{Name: "r, revision", Usage: revisionUsage},
cli.StringFlag{Name: "e, endpoint", Usage: endpointUsage},
cli.BoolFlag{Name: "l, locators", Usage: "upload data in locator format instead of JSON"},
cli.StringFlag{Name: "d, data", Usage: "the user-provided build data to upload"},
cli.BoolFlag{Name: "debug", Usage: debugUsage},
},
Expand Down Expand Up @@ -232,7 +233,11 @@ func defaultCmd(c *cli.Context) {
s.Suffix = fmt.Sprintf(" Running module analysis (%d/%d): %s", i+1, len(conf.Modules), m.Path)
s.Restart()
deps, err := builder.Analyze(module, conf.AnalyzeCmd.AllowUnresolved)
mainLogger.Debugf("Analysis complete: %#v", deps)
if err != nil {
mainLogger.Warningf("Analysis failed for module %s: %s", module.Name, err.Error())
} else {
mainLogger.Debugf("Analysis complete: %#v", deps)
}
s.Stop()

dependencies[analysisKey{
Expand Down
22 changes: 19 additions & 3 deletions cmd/fossa/upload.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,25 @@ func uploadCmd(c *cli.Context) {
}

var data []normalizedModule
err = json.Unmarshal([]byte(conf.UploadCmd.Data), &data)
if err != nil {
uploadLogger.Fatalf("Could not parse user-provided build data: %s", err.Error())
if conf.UploadCmd.Locators {
var deps []normalizedDependency
lines := strings.Split(conf.UploadCmd.Data, "\n")
for _, line := range lines {
deps = append(deps, normalizedDependency{
Locator: line,
})
}
data = append(data, normalizedModule{
Build: normalizedBuild{
Succeeded: true,
Dependencies: deps,
},
})
} else {
err = json.Unmarshal([]byte(conf.UploadCmd.Data), &data)
if err != nil {
uploadLogger.Fatalf("Could not parse user-provided build data: %s", err.Error())
}
}

msg, err := doUpload(conf, data)
Expand Down
6 changes: 4 additions & 2 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ type TestConfig struct {

// UploadConfig specifies the config for the upload cmd
type UploadConfig struct {
Data string
Locators bool
Data string
}

// CLIConfig specifies the config available to the cli
Expand Down Expand Up @@ -133,7 +134,8 @@ func New(c *cli.Context) (CLIConfig, error) {
},

UploadCmd: UploadConfig{
Data: c.String("data"),
Locators: c.Bool("locators"),
Data: c.String("data"),
},
}

Expand Down
7 changes: 6 additions & 1 deletion config/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ const (
Maven = ModuleType("maven")
// SBT is the module type for scala-sbt.org
SBT = ModuleType("sbt")
// Gradle is the module type for gradle.org
Gradle = ModuleType("gradle")

// Ecosystems where many tools behave similarly

Expand All @@ -37,7 +39,7 @@ const (
)

// ModuleTypes holds the list of all available module types for analysis
var ModuleTypes = []ModuleType{Bower, Composer, Maven, SBT, Ruby, Nodejs, Golang, VendoredArchives}
var ModuleTypes = []ModuleType{Bower, Composer, Maven, SBT, Gradle, Ruby, Nodejs, Golang, VendoredArchives}

// GetModuleType returns a ModuleType for a variety of config keys
func GetModuleType(configKey string) ModuleType {
Expand Down Expand Up @@ -88,6 +90,9 @@ func GetModuleType(configKey string) ModuleType {
case "sbt":
return SBT

case "gradle":
return Gradle

// Archive aliases
case "vendoredarchives":
return VendoredArchives
Expand Down
39 changes: 22 additions & 17 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ RUN sudo apt-get update && \

# Install Node.js runtime
RUN wget https://nodejs.org/dist/v8.9.4/node-v8.9.4-linux-x64.tar.xz -O /tmp/node.tar.xz && \
sudo tar -xJf /tmp/node.tar.xz -C /usr/local --strip-components=1 --no-same-owner && \
sudo tar -xf /tmp/node.tar.xz -C /usr/local --strip-components=1 --no-same-owner && \
sudo ln -s /usr/local/bin/node /usr/local/bin/nodejs && \
mkdir $HOME/.npm && \
npm config set prefix $HOME/.npm
Expand All @@ -36,7 +36,7 @@ RUN sudo gem install bundler

# Install Go compiler
RUN wget https://dl.google.com/go/go1.9.4.linux-amd64.tar.gz -O /tmp/go.tar.gz && \
sudo tar -C /usr/local -xzf /tmp/go.tar.gz
sudo tar -xf /tmp/go.tar.gz -C /usr/local
ENV GOPATH=/home/fossa/go PATH=$PATH:/usr/local/go/bin:/home/fossa/go/bin

# Install Go build tools
Expand All @@ -59,23 +59,28 @@ RUN mkdir -p $HOME/go/bin && \
RUN sudo apt-get install -y php5

# Install PHP build tools
RUN curl -sS https://getcomposer.org/installer | sudo php -- --install-dir=/usr/local/bin --filename=composer
RUN curl https://getcomposer.org/installer | sudo php -- --install-dir=/usr/local/bin --filename=composer

# Install Java runtime
RUN sudo add-apt-repository -y ppa:openjdk-r/ppa && \
sudo apt-get update && \
sudo apt-get install -y openjdk-8-jdk && \
sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
# Install JVM version manager
RUN sudo apt-get install -y zip unzip && \
curl https://get.sdkman.io | bash

# Install Java build tools
RUN sudo apt-get install -y maven
# Install JVM runtimes and build tools
RUN ["/bin/bash", "-c", "\
source $HOME/.sdkman/bin/sdkman-init.sh && \
# Install Java runtime and build tools
sdk install java && \
sdk install maven && \
sdk install gradle && \
# Install Scala runtime and build tools
sdk install scala && \
sdk install sbt \
"]

# Install Scala build tools
RUN echo "deb https://dl.bintray.com/sbt/debian /" | sudo tee -a /etc/apt/sources.list.d/sbt.list && \
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 && \
sudo apt-get update && \
sudo apt-get install -y sbt && \
sudo /var/lib/dpkg/info/ca-certificates-java.postinst configure && \
sbt about
# Install Android SDK
RUN wget https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip -O /tmp/sdk-tools-linux.zip && \
sudo unzip /tmp/sdk-tools-linux.zip -d /opt/android-sdk && \
sudo chmod -R 775 /opt/android-sdk
ENV PATH=$PATH:/opt/android-sdk/tools/bin ANDROID_HOME=/opt/android-sdk

CMD [ "/bin/bash" ]
Loading

0 comments on commit 3105635

Please sign in to comment.