Skip to content
This repository has been archived by the owner on Sep 9, 2020. It is now read-only.

Commit

Permalink
WIP: implement pruning
Browse files Browse the repository at this point in the history 
Signed-off-by: Ibrahim AshShohail <ibra.sho@gmail.com>
  • Loading branch information
@ibrasho
ibrasho committed Aug 5, 2017
1 parent 5c34ae8 commit 54ccce4
Show file tree
Hide file tree
Showing 4 changed files with 395 additions and 72 deletions.
287 changes: 285 additions & 2 deletions internal/gps/prune.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,300 @@

package gps

import (
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"sort"
"strings"

"github.com/pkg/errors"
)

// PruneOptions represents the pruning options used to write the dependecy tree.
type PruneOptions uint8

const (
// PruneNestedVendorDirs indicates if nested vendor directories should be pruned.
PruneNestedVendorDirs = 1 << iota
// PruneUnusedPackages indicates if unused Go packages should be pruned.
PruneUnusedPackages
// PruneNonGoFiles indicates if non-Go files should be pruned.
// LICENSE & COPYING files are kept for convience.
PruneNonGoFiles
// PruneGoTestFiles indicates if Go test files should be pruned.
PruneGoTestFiles
// PruneUnusedPackages indicates if unused Go packages should be pruned.
PruneUnusedPackages
)

var (
preservedNonGoFiles = []string{
"LICENSE",
"COPYING",
}
)

func Prune(baseDir string, l Lock, options PruneOptions, logger *log.Logger) error {
if (options & PruneNestedVendorDirs) != 0 {
if err := pruneNestedVendorDirs(baseDir); err != nil {
return err
}
}

if (options & PruneUnusedPackages) != 0 {
if l == nil {
return errors.New("pruning unused packages requires passing non-nil Lock")
}
if err := pruneUnusedPackages(baseDir, l, logger); err != nil {
return err
}
}

// if (options & PruneNonGoFiles) != 0 {
// if err := pruneNonGoFiles(baseDir, logger); err != nil {
// return err
// }
// }

// if (options & PruneGoTestFiles) != 0 {
// if err := pruneGoTestFiles(baseDir, logger); err != nil {
// return err
// }
// }

// Delete all empty directories.
return errors.Wrap(pruneEmptyDirs(baseDir, logger), "could not prune empty dirs")
}

func pruneNestedVendorDirs(baseDir string) error {
return filepath.Walk(baseDir, stripNestedVendorDirs(baseDir))
}

func pruneUnusedPackages(baseDir string, l Lock, logger *log.Logger) error {
unused, err := calculateUnusedPackages(baseDir, l, logger)
if err != nil {
return errors.Wrap(err, "could not strip unused packages")
}

for _, pkg := range unused {
pkgPath := filepath.Join(baseDir, pkg)

files, err := ioutil.ReadDir(pkgPath)
if err != nil {
return err
}

// Delete *.go files in the package directory.
for _, file := range files {
// Skip directories and files that don't have a .go suffix.
if file.IsDir() || !strings.HasSuffix(file.Name(), ".go") {
continue
}

if err := os.Remove(filepath.Join(pkgPath, file.Name())); err != nil {
return err
}
}
}

return nil
}

func calculateUnusedPackages(baseDir string, l Lock, logger *log.Logger) ([]string, error) {
imported := calculateImportedPackages(l)
sort.Strings(imported)

var unused []string

if logger != nil {
logger.Println("Calculating unused packages to prune. Checking the following packages:")
}

err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}

fmt.Println(info.Name(), info.IsDir(), info.Mode())

// Ignore baseDir and anything that's not a directory.
if path == baseDir || !info.IsDir() {
return nil
}

pkg := strings.TrimPrefix(path, baseDir+string(filepath.Separator))
if logger != nil {
logger.Printf(" %s", pkg)
}

// If pkg is not a parent of an imported package, add it to the unused list.
i := sort.Search(len(imported), func(i int) bool {
return pkg <= imported[i]
})
if i >= len(imported) || !strings.HasPrefix(imported[i], pkg) {
unused = append(unused, path)
}

return nil
})
fmt.Println("err", err)

return unused, err
}

func calculateImportedPackages(l Lock) []string {
var imported []string

for _, project := range l.Projects() {
projectRoot := string(project.Ident().ProjectRoot)
for _, pkg := range project.Packages() {
imported = append(imported, filepath.Join(projectRoot, pkg))
}
}
return imported
}

func pruneNonGoFiles(baseDir string, logger *log.Logger) error {
files, err := calculateNonGoFiles(baseDir)
if err != nil {
return errors.Wrap(err, "could not prune non-Go files")
}

if err := deleteFiles(files); err != nil {
return err
}

return nil
}

func calculateNonGoFiles(baseDir string) ([]string, error) {
var files []string

err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}

// Ignore directories.
if info.IsDir() {
return nil
}

// Ignore all Go files.
if strings.HasSuffix(info.Name(), ".go") {
return nil
}

// Ignore preserved non-Go files. We check for prefix incase the file
// has an extension. For example: LICENSE.md.
for _, prefix := range preservedNonGoFiles {
if strings.HasPrefix(info.Name(), prefix) {
return nil
}
}

files = append(files, path)

return nil
})

return files, err
}

func pruneGoTestFiles(baseDir string, logger *log.Logger) error {
files, err := calculateGoTestFiles(baseDir)
if err != nil {
return errors.Wrap(err, "could not prune Go test files")
}

if err := deleteFiles(files); err != nil {
return err
}

return nil
}

func calculateGoTestFiles(baseDir string) ([]string, error) {
var files []string

err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}

// Ignore directories.
if info.IsDir() {
return nil
}

// Ignore any files that is not a Go test file.
if !strings.HasSuffix(info.Name(), "_test.go") {
return nil
}

files = append(files, path)

return nil
})

return files, err
}

func deleteFiles(paths []string) error {
for _, path := range paths {
if err := os.Remove(path); err != nil {
return err
}
}
return nil
}

func pruneEmptyDirs(baseDir string, logger *log.Logger) error {
empty, err := calculateEmptyDirs(baseDir)
if err != nil {
return err
}

if logger != nil {
logger.Println("Deleting empty directories:")
}

for _, dir := range empty {
if logger != nil {
logger.Printf(" %s\n", strings.TrimPrefix(dir, baseDir+string(os.PathSeparator)))
}
if err := os.Remove(dir); err != nil {
return err
}
}

return nil
}
func calculateEmptyDirs(baseDir string) ([]string, error) {
var empty []string

err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil
}

if !info.IsDir() {
return nil
}

// TODO(ibrasho) should we use fs.IsNonEmptyDir instead?
files, err := ioutil.ReadDir(path)
if err != nil {
return err
}

if len(files) == 0 {
empty = append(empty, path)
}

return nil
})

return empty, err
}
56 changes: 33 additions & 23 deletions internal/gps/source_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"context"
"encoding/hex"
"fmt"
"log"
"os"
"os/signal"
"path/filepath"
Expand Down Expand Up @@ -486,41 +487,50 @@ func (sm *SourceMgr) ExportProject(id ProjectIdentifier, v Version, to string) e
return srcg.exportVersionTo(context.TODO(), v, to)
}

// WriteDepTree takes a basedir, a Lock and a PruneOptions, and exports all the
// projects listed in the lock to the appropriate target location within basedir.
// WriteDepTree takes a baseDir, a Lock and a PruneOptions, and exports all the
// projects listed in the lock to the appropriate target location within baseDir.
//
// If the goal is to populate a vendor directory, basedir should be the absolute
// If the goal is to populate a vendor directory, baseDir should be the absolute
// path to that vendor directory, not its parent (a project root, typically).
func (sm *SourceMgr) WriteDepTree(basedir string, l Lock, options PruneOptions) error {
func (sm *SourceMgr) WriteDepTree(baseDir string, l Lock, options PruneOptions) error {
if l == nil {
return fmt.Errorf("must provide non-nil Lock to WriteDepTree")
}

if err := os.MkdirAll(basedir, 0777); err != nil {
if err := os.MkdirAll(baseDir, 0777); err != nil {
return err
}

// TODO(sdboyer) parallelize
var wg sync.WaitGroup
errCh := make(chan error, len(l.Projects()))

for _, p := range l.Projects() {
to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot)))
wg.Add(1)
go func(p LockedProject) {
to := filepath.FromSlash(filepath.Join(baseDir, string(p.Ident().ProjectRoot)))

if err := sm.ExportProject(p.Ident(), p.Version(), to); err != nil {
removeAll(basedir)
return fmt.Errorf("error while exporting %s: %s", p.Ident().ProjectRoot, err)
}
if err := sm.ExportProject(p.Ident(), p.Version(), to); err != nil {
removeAll(baseDir)
errCh <- errors.Wrapf(err, "error while exporting %s: %s", p.Ident().ProjectRoot)
}

if (options & PruneNestedVendorDirs) != 0 {
filepath.Walk(to, stripVendor)
}
if (options & PruneNonGoFiles) != 0 {
// TODO: prune non Go files
}
if (options & PruneGoTestFiles) != 0 {
// TODO: prune Go test files
}
if (options & PruneUnusedPackages) != 0 {
// TODO: prune unused packages
}
wg.Done()
}(p)
}

wg.Wait()

// TODO(ibrasho) handle multiple errors
if len(errCh) > 0 {
return <-errCh
}

// TODO(ibrasho) pass a proper logger?
logger := log.New(os.Stdout, "", 0)

if err := Prune(baseDir, l, options, logger); err != nil {
// removeAll(baseDir)
return err
}

return nil
Expand Down
Loading

0 comments on commit 54ccce4

Please sign in to comment.