From ddaa9a67353312a01c585a3067d718b2187e203f Mon Sep 17 00:00:00 2001 From: gatesvp Date: Sun, 3 May 2015 22:59:27 -0700 Subject: [PATCH 01/22] Make ipfs add ignore files by default. --- commands/files/is_hidden.go | 19 +++++++++ commands/files/is_hidden_windows.go | 29 +++++++++++++ commands/files/serialfile.go | 2 +- core/commands/add.go | 66 +++++++++++++++++------------ 4 files changed, 89 insertions(+), 27 deletions(-) create mode 100644 commands/files/is_hidden.go create mode 100644 commands/files/is_hidden_windows.go diff --git a/commands/files/is_hidden.go b/commands/files/is_hidden.go new file mode 100644 index 00000000000..004ec04e51a --- /dev/null +++ b/commands/files/is_hidden.go @@ -0,0 +1,19 @@ +// +build !windows + +package files + +import ( + "path/filepath" + "strings" +) + +func IsHidden(f File) bool { + + fName := filepath.Base(f.FileName()) + + if strings.HasPrefix(fName, ".") { + return true, nil + } + + return false, nil +} diff --git a/commands/files/is_hidden_windows.go b/commands/files/is_hidden_windows.go new file mode 100644 index 00000000000..2f310d4448a --- /dev/null +++ b/commands/files/is_hidden_windows.go @@ -0,0 +1,29 @@ +// +build windows + +package files + +import ( + "path/filepath" + "strings" + "syscall" +) + +func IsHidden(f File) bool { + + fName := filepath.Base(f.FileName()) + + if strings.HasPrefix(fName, ".") { + return true + } + + p, e := syscall.UTF16PtrFromString(f.FileName()) + if e != nil { + return false + } + + attrs, e := syscall.GetFileAttributes(p) + if e != nil { + return false + } + return attrs&syscall.FILE_ATTRIBUTE_HIDDEN != 0, nil +} diff --git a/commands/files/serialfile.go b/commands/files/serialfile.go index aeba01fa7ed..461bde33600 100644 --- a/commands/files/serialfile.go +++ b/commands/files/serialfile.go @@ -3,7 +3,7 @@ package files import ( "io" "os" - fp "path" + fp "path/filepath" "sort" "syscall" ) diff --git a/core/commands/add.go b/core/commands/add.go index 85692337b7f..501b94a0820 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -29,6 +29,7 @@ const progressReaderIncrement = 1024 * 256 const ( progressOptionName = "progress" wrapOptionName = "wrap-with-directory" + hiddenOptionName = "hidden" ) type AddedObject struct { @@ -57,6 +58,7 @@ remains to be implemented. cmds.BoolOption(progressOptionName, "p", "Stream progress data"), cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object"), cmds.BoolOption("t", "trickle", "Use trickle-dag format for dag generation"), + cmds.BoolOption(hiddenOptionName, "Include files that are hidden"), }, PreRun: func(req cmds.Request) error { if quiet, _, _ := req.Option("quiet").Bool(); quiet { @@ -90,6 +92,7 @@ remains to be implemented. progress, _, _ := req.Option(progressOptionName).Bool() wrap, _, _ := req.Option(wrapOptionName).Bool() + hidden, _, _ := req.Option(hiddenOptionName).Bool() outChan := make(chan interface{}) res.SetOutput((<-chan interface{})(outChan)) @@ -107,7 +110,7 @@ remains to be implemented. return } - rootnd, err := addFile(n, file, outChan, progress, wrap) + rootnd, err := addFile(n, file, outChan, progress, wrap, hidden) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -227,9 +230,12 @@ func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { return node, nil } -func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool, wrap bool) (*dag.Node, error) { +func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool, wrap bool, hidden bool) (*dag.Node, error) { if file.IsDirectory() { - return addDir(n, file, out, progress) + return addDir(n, file, out, progress, hidden) + } else if fileIsHidden := files.IsHidden(file); fileIsHidden && !hidden { + log.Infof("%s is a hidden file, skipping", file.FileName()) + return nil, &hiddenFileError{file.FileName()} } // if the progress flag was specified, wrap the file so that we can send @@ -263,43 +269,51 @@ func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress b return dagnode, nil } -func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool) (*dag.Node, error) { - log.Infof("adding directory: %s", dir.FileName()) +func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool, hidden bool) (*dag.Node, error) { tree := &dag.Node{Data: ft.FolderPBData()} + if dirIsHidden := files.IsHidden(dir); dirIsHidden && !hidden { + log.Infof("ignoring directory: %s", dir.FileName()) + } else { + log.Infof("adding directory: %s", dir.FileName()) + for { + file, err := dir.NextFile() + if err != nil && err != io.EOF { + return nil, err + } + if file == nil { + break + } - for { - file, err := dir.NextFile() - if err != nil && err != io.EOF { - return nil, err - } - if file == nil { - break + node, err := addFile(n, file, out, progress, false, hidden) + if _, ok := err.(*hiddenFileError); ok { + // hidden file error, set the node to nil for below + node = nil + } else if err != nil { + return nil, err + } + + if node != nil { + _, name := path.Split(file.FileName()) + + err = tree.AddNodeLink(name, node) + if err != nil { + return nil, err + } + } } - node, err := addFile(n, file, out, progress, false) + err := outputDagnode(out, dir.FileName(), tree) if err != nil { return nil, err } - _, name := path.Split(file.FileName()) - - err = tree.AddNodeLink(name, node) + _, err = n.DAG.Add(tree) if err != nil { return nil, err } } - err := outputDagnode(out, dir.FileName(), tree) - if err != nil { - return nil, err - } - - _, err = n.DAG.Add(tree) - if err != nil { - return nil, err - } - return tree, nil } From 9517911f1b5c28621329302fad265d4eac84f97c Mon Sep 17 00:00:00 2001 From: gatesvp Date: Sun, 3 May 2015 23:10:24 -0700 Subject: [PATCH 02/22] Fix my build --- commands/files/is_hidden_windows.go | 2 +- core/commands/add.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/commands/files/is_hidden_windows.go b/commands/files/is_hidden_windows.go index 2f310d4448a..844610fb8ec 100644 --- a/commands/files/is_hidden_windows.go +++ b/commands/files/is_hidden_windows.go @@ -25,5 +25,5 @@ func IsHidden(f File) bool { if e != nil { return false } - return attrs&syscall.FILE_ATTRIBUTE_HIDDEN != 0, nil + return attrs&syscall.FILE_ATTRIBUTE_HIDDEN != 0 } diff --git a/core/commands/add.go b/core/commands/add.go index 501b94a0820..3bad0983267 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -332,6 +332,14 @@ func outputDagnode(out chan interface{}, name string, dn *dag.Node) error { return nil } +type hiddenFileError struct { + fileName string +} + +func (e *hiddenFileError) Error() string { + return fmt.Sprintf("%s is a hidden file", e.fileName) +} + type progressReader struct { file files.File out chan interface{} From 50e3b151813bcc254b897993338eeaebdf67999d Mon Sep 17 00:00:00 2001 From: gatesvp Date: Wed, 6 May 2015 00:33:17 -0700 Subject: [PATCH 03/22] ipfs add now respects .ipfsignore files --- commands/files/is_hidden.go | 2 +- commands/files/is_hidden_windows.go | 2 +- core/commands/add.go | 130 +++++++++++++++++++--------- 3 files changed, 93 insertions(+), 41 deletions(-) diff --git a/commands/files/is_hidden.go b/commands/files/is_hidden.go index 004ec04e51a..0aeb6ba516e 100644 --- a/commands/files/is_hidden.go +++ b/commands/files/is_hidden.go @@ -11,7 +11,7 @@ func IsHidden(f File) bool { fName := filepath.Base(f.FileName()) - if strings.HasPrefix(fName, ".") { + if strings.HasPrefix(fName, ".") && len(fName) > 1 { return true, nil } diff --git a/commands/files/is_hidden_windows.go b/commands/files/is_hidden_windows.go index 844610fb8ec..5d263931033 100644 --- a/commands/files/is_hidden_windows.go +++ b/commands/files/is_hidden_windows.go @@ -12,7 +12,7 @@ func IsHidden(f File) bool { fName := filepath.Base(f.FileName()) - if strings.HasPrefix(fName, ".") { + if strings.HasPrefix(fName, ".") && len(fName) > 1 { return true } diff --git a/core/commands/add.go b/core/commands/add.go index 3bad0983267..46ce7dbcab2 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -3,11 +3,13 @@ package commands import ( "fmt" "io" + "os" "path" "strings" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + ignore "github.com/sabhiram/go-git-ignore" cmds "github.com/ipfs/go-ipfs/commands" files "github.com/ipfs/go-ipfs/commands/files" @@ -94,6 +96,16 @@ remains to be implemented. wrap, _, _ := req.Option(wrapOptionName).Bool() hidden, _, _ := req.Option(hiddenOptionName).Bool() + var ignoreFilePatterns []ignore.GitIgnore + + // Check the $IPFS_PATH + if ipfs_path := os.Getenv("$IPFS_PATH"); len(ipfs_path) > 0 { + baseFilePattern, err := ignore.CompileIgnoreFile(path.Join(ipfs_path, ".ipfsignore")) + if err == nil && baseFilePattern != nil { + ignoreFilePatterns = append(ignoreFilePatterns, *baseFilePattern) + } + } + outChan := make(chan interface{}) res.SetOutput((<-chan interface{})(outChan)) @@ -110,7 +122,7 @@ remains to be implemented. return } - rootnd, err := addFile(n, file, outChan, progress, wrap, hidden) + rootnd, err := addFile(n, file, outChan, progress, wrap, hidden, ignoreFilePatterns) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -230,14 +242,26 @@ func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { return node, nil } -func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool, wrap bool, hidden bool) (*dag.Node, error) { - if file.IsDirectory() { - return addDir(n, file, out, progress, hidden) - } else if fileIsHidden := files.IsHidden(file); fileIsHidden && !hidden { - log.Infof("%s is a hidden file, skipping", file.FileName()) +func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool, wrap bool, hidden bool, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { + // Check if file is hidden + if fileIsHidden := files.IsHidden(file); fileIsHidden && !hidden { + log.Debugf("%s is hidden, skipping", file.FileName()) return nil, &hiddenFileError{file.FileName()} } + // Check for ignore files matches + for i := range ignoreFilePatterns { + if ignoreFilePatterns[i].MatchesPath(file.FileName()) { + log.Debugf("%s is ignored file, skipping", file.FileName()) + return nil, &ignoreFileError{file.FileName()} + } + } + + // Check if "file" is actually a directory + if file.IsDirectory() { + return addDir(n, file, out, progress, hidden, ignoreFilePatterns) + } + // if the progress flag was specified, wrap the file so that we can send // progress updates to the client (over the output channel) var reader io.Reader = file @@ -269,54 +293,74 @@ func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress b return dagnode, nil } -func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool, hidden bool) (*dag.Node, error) { +func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool, hidden bool, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { tree := &dag.Node{Data: ft.FolderPBData()} - if dirIsHidden := files.IsHidden(dir); dirIsHidden && !hidden { - log.Infof("ignoring directory: %s", dir.FileName()) - } else { - log.Infof("adding directory: %s", dir.FileName()) - for { - file, err := dir.NextFile() - if err != nil && err != io.EOF { - return nil, err - } - if file == nil { - break - } - - node, err := addFile(n, file, out, progress, false, hidden) - if _, ok := err.(*hiddenFileError); ok { - // hidden file error, set the node to nil for below - node = nil - } else if err != nil { - return nil, err - } + log.Infof("adding directory: %s", dir.FileName()) - if node != nil { - _, name := path.Split(file.FileName()) + // Check for an .ipfsignore file that is local to this Dir and append to the incoming + localIgnorePatterns := checkForLocalIgnorePatterns(dir, ignoreFilePatterns) - err = tree.AddNodeLink(name, node) - if err != nil { - return nil, err - } - } + for { + file, err := dir.NextFile() + if err != nil && err != io.EOF { + return nil, err + } + if file == nil { + break } - err := outputDagnode(out, dir.FileName(), tree) - if err != nil { + node, err := addFile(n, file, out, progress, false, hidden, localIgnorePatterns) + if _, ok := err.(*hiddenFileError); ok { + // hidden file error, set the node to nil for below + node = nil + } else if _, ok := err.(*ignoreFileError); ok { + // ignore file error, set the node to nil for below + node = nil + } else if err != nil { return nil, err } - _, err = n.DAG.Add(tree) - if err != nil { - return nil, err + if node != nil { + _, name := path.Split(file.FileName()) + + err = tree.AddNodeLink(name, node) + if err != nil { + return nil, err + } } } + err := outputDagnode(out, dir.FileName(), tree) + if err != nil { + return nil, err + } + + _, err = n.DAG.Add(tree) + if err != nil { + return nil, err + } + return tree, nil } +func checkForLocalIgnorePatterns(dir files.File, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore { + var ignorePathname string + if dir.FileName() == "." { + ignorePathname = ".ipfsignore" + } else { + ignorePathname = path.Join(dir.FileName(), ".ipfsignore") + } + + localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname) + if ignoreErr == nil && localIgnore != nil { + log.Debugf("found ignore file: %s", dir.FileName()) + return append(ignoreFilePatterns, *localIgnore) + } else { + return ignoreFilePatterns + } +} + // outputDagnode sends dagnode info over the output channel func outputDagnode(out chan interface{}, name string, dn *dag.Node) error { o, err := getOutput(dn) @@ -340,6 +384,14 @@ func (e *hiddenFileError) Error() string { return fmt.Sprintf("%s is a hidden file", e.fileName) } +type ignoreFileError struct { + fileName string +} + +func (e *ignoreFileError) Error() string { + return fmt.Sprintf("%s is an ignored file", e.fileName) +} + type progressReader struct { file files.File out chan interface{} From efae6b1fc70f41ae04115d7d6770192a504b3425 Mon Sep 17 00:00:00 2001 From: gatesvp Date: Wed, 6 May 2015 14:51:59 -0700 Subject: [PATCH 04/22] fix ipfs add to ensure that we are checking parent paths for appropriate .ipfsignore files --- core/commands/add.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 46ce7dbcab2..a1d7233b21b 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -122,7 +122,15 @@ remains to be implemented. return } - rootnd, err := addFile(n, file, outChan, progress, wrap, hidden, ignoreFilePatterns) + // If the file is not a folder, then let's get the root of that + // folder and attempt to load the appropriate .ipfsignore. + localIgnorePatterns := ignoreFilePatterns + if !file.IsDirectory() { + parentPath := path.Dir(file.FileName()) + localIgnorePatterns = checkForLocalIgnorePatterns(parentPath, ignoreFilePatterns) + } + + rootnd, err := addFile(n, file, outChan, progress, wrap, hidden, localIgnorePatterns) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -299,7 +307,7 @@ func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress boo log.Infof("adding directory: %s", dir.FileName()) // Check for an .ipfsignore file that is local to this Dir and append to the incoming - localIgnorePatterns := checkForLocalIgnorePatterns(dir, ignoreFilePatterns) + localIgnorePatterns := checkForLocalIgnorePatterns(dir.FileName(), ignoreFilePatterns) for { file, err := dir.NextFile() @@ -344,17 +352,17 @@ func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress boo return tree, nil } -func checkForLocalIgnorePatterns(dir files.File, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore { +func checkForLocalIgnorePatterns(dir string, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore { var ignorePathname string - if dir.FileName() == "." { + if dir == "." { ignorePathname = ".ipfsignore" } else { - ignorePathname = path.Join(dir.FileName(), ".ipfsignore") + ignorePathname = path.Join(dir, ".ipfsignore") } localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname) if ignoreErr == nil && localIgnore != nil { - log.Debugf("found ignore file: %s", dir.FileName()) + log.Debugf("found ignore file: %s", dir) return append(ignoreFilePatterns, *localIgnore) } else { return ignoreFilePatterns From 1e66056f81748296f8f77599f61580deb1a13556 Mon Sep 17 00:00:00 2001 From: gatesvp Date: Fri, 8 May 2015 13:17:45 -0700 Subject: [PATCH 05/22] Clean-up based on PR feedback https://github.com/ipfs/go-ipfs/pull/1204 --- core/commands/add.go | 73 +++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index a1d7233b21b..382163ae6c5 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -3,7 +3,6 @@ package commands import ( "fmt" "io" - "os" "path" "strings" @@ -98,8 +97,8 @@ remains to be implemented. var ignoreFilePatterns []ignore.GitIgnore - // Check the $IPFS_PATH - if ipfs_path := os.Getenv("$IPFS_PATH"); len(ipfs_path) > 0 { + // Check the IPFS_PATH + if ipfs_path := req.Context().ConfigRoot; len(ipfs_path) > 0 { baseFilePattern, err := ignore.CompileIgnoreFile(path.Join(ipfs_path, ".ipfsignore")) if err == nil && baseFilePattern != nil { ignoreFilePatterns = append(ignoreFilePatterns, *baseFilePattern) @@ -130,7 +129,7 @@ remains to be implemented. localIgnorePatterns = checkForLocalIgnorePatterns(parentPath, ignoreFilePatterns) } - rootnd, err := addFile(n, file, outChan, progress, wrap, hidden, localIgnorePatterns) + rootnd, err := addFile(n, addParams{file, outChan, progress, wrap, hidden, localIgnorePatterns}) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -236,6 +235,15 @@ remains to be implemented. Type: AddedObject{}, } +type addParams struct { + file files.File + out chan interface{} + progress bool + wrap bool + hidden bool + ignoreFilePatterns []ignore.GitIgnore +} + func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { node, err := importer.BuildDagFromReader(reader, n.DAG, nil, chunk.DefaultSplitter) if err != nil { @@ -250,41 +258,41 @@ func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { return node, nil } -func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool, wrap bool, hidden bool, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { +func addFile(n *core.IpfsNode, params addParams) (*dag.Node, error) { // Check if file is hidden - if fileIsHidden := files.IsHidden(file); fileIsHidden && !hidden { - log.Debugf("%s is hidden, skipping", file.FileName()) - return nil, &hiddenFileError{file.FileName()} + if fileIsHidden := files.IsHidden(params.file); fileIsHidden && !params.hidden { + log.Debugf("%s is hidden, skipping", params.file.FileName()) + return nil, &hiddenFileError{params.file.FileName()} } // Check for ignore files matches - for i := range ignoreFilePatterns { - if ignoreFilePatterns[i].MatchesPath(file.FileName()) { - log.Debugf("%s is ignored file, skipping", file.FileName()) - return nil, &ignoreFileError{file.FileName()} + for i := range params.ignoreFilePatterns { + if params.ignoreFilePatterns[i].MatchesPath(params.file.FileName()) { + log.Debugf("%s is ignored file, skipping", params.file.FileName()) + return nil, &ignoreFileError{params.file.FileName()} } } // Check if "file" is actually a directory - if file.IsDirectory() { - return addDir(n, file, out, progress, hidden, ignoreFilePatterns) + if params.file.IsDirectory() { + return addDir(n, params) } // if the progress flag was specified, wrap the file so that we can send // progress updates to the client (over the output channel) - var reader io.Reader = file - if progress { - reader = &progressReader{file: file, out: out} + var reader io.Reader = params.file + if params.progress { + reader = &progressReader{file: params.file, out: params.out} } - if wrap { - p, dagnode, err := coreunix.AddWrapped(n, reader, path.Base(file.FileName())) + if params.wrap { + p, dagnode, err := coreunix.AddWrapped(n, reader, path.Base(params.file.FileName())) if err != nil { return nil, err } - out <- &AddedObject{ + params.out <- &AddedObject{ Hash: p, - Name: file.FileName(), + Name: params.file.FileName(), } return dagnode, nil } @@ -294,23 +302,23 @@ func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress b return nil, err } - log.Infof("adding file: %s", file.FileName()) - if err := outputDagnode(out, file.FileName(), dagnode); err != nil { + log.Infof("adding file: %s", params.file.FileName()) + if err := outputDagnode(params.out, params.file.FileName(), dagnode); err != nil { return nil, err } return dagnode, nil } -func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool, hidden bool, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { +func addDir(n *core.IpfsNode, params addParams) (*dag.Node, error) { tree := &dag.Node{Data: ft.FolderPBData()} - log.Infof("adding directory: %s", dir.FileName()) + log.Infof("adding directory: %s", params.file.FileName()) // Check for an .ipfsignore file that is local to this Dir and append to the incoming - localIgnorePatterns := checkForLocalIgnorePatterns(dir.FileName(), ignoreFilePatterns) + localIgnorePatterns := checkForLocalIgnorePatterns(params.file.FileName(), params.ignoreFilePatterns) for { - file, err := dir.NextFile() + file, err := params.file.NextFile() if err != nil && err != io.EOF { return nil, err } @@ -318,7 +326,7 @@ func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress boo break } - node, err := addFile(n, file, out, progress, false, hidden, localIgnorePatterns) + node, err := addFile(n, addParams{file, params.out, params.progress, false, params.hidden, localIgnorePatterns}) if _, ok := err.(*hiddenFileError); ok { // hidden file error, set the node to nil for below node = nil @@ -339,7 +347,7 @@ func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress boo } } - err := outputDagnode(out, dir.FileName(), tree) + err := outputDagnode(params.out, params.file.FileName(), tree) if err != nil { return nil, err } @@ -353,12 +361,7 @@ func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress boo } func checkForLocalIgnorePatterns(dir string, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore { - var ignorePathname string - if dir == "." { - ignorePathname = ".ipfsignore" - } else { - ignorePathname = path.Join(dir, ".ipfsignore") - } + ignorePathname := path.Join(dir, ".ipfsignore") localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname) if ignoreErr == nil && localIgnore != nil { From 66a5f1f1f87a25b7036a35d9e758914213f415f1 Mon Sep 17 00:00:00 2001 From: gatesvp Date: Tue, 12 May 2015 20:49:53 -0700 Subject: [PATCH 06/22] Clean-up add CLI. Basic params are a struct and addFile/addDir are now methods on the struct --- core/commands/add.go | 64 ++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 382163ae6c5..46c06c76f42 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -129,7 +129,8 @@ remains to be implemented. localIgnorePatterns = checkForLocalIgnorePatterns(parentPath, ignoreFilePatterns) } - rootnd, err := addFile(n, addParams{file, outChan, progress, wrap, hidden, localIgnorePatterns}) + addParams := adder{n, outChan, progress, wrap, hidden} + rootnd, err := addParams.addFile(file, localIgnorePatterns) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -235,13 +236,12 @@ remains to be implemented. Type: AddedObject{}, } -type addParams struct { - file files.File - out chan interface{} - progress bool - wrap bool - hidden bool - ignoreFilePatterns []ignore.GitIgnore +type adder struct { + node *core.IpfsNode + out chan interface{} + progress bool + wrap bool + hidden bool } func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { @@ -258,67 +258,67 @@ func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { return node, nil } -func addFile(n *core.IpfsNode, params addParams) (*dag.Node, error) { +func (params *adder) addFile(file files.File, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { // Check if file is hidden - if fileIsHidden := files.IsHidden(params.file); fileIsHidden && !params.hidden { - log.Debugf("%s is hidden, skipping", params.file.FileName()) - return nil, &hiddenFileError{params.file.FileName()} + if fileIsHidden := files.IsHidden(file); fileIsHidden && !params.hidden { + log.Debugf("%s is hidden, skipping", file.FileName()) + return nil, &hiddenFileError{file.FileName()} } // Check for ignore files matches - for i := range params.ignoreFilePatterns { - if params.ignoreFilePatterns[i].MatchesPath(params.file.FileName()) { - log.Debugf("%s is ignored file, skipping", params.file.FileName()) - return nil, &ignoreFileError{params.file.FileName()} + for i := range ignoreFilePatterns { + if ignoreFilePatterns[i].MatchesPath(file.FileName()) { + log.Debugf("%s is ignored file, skipping", file.FileName()) + return nil, &ignoreFileError{file.FileName()} } } // Check if "file" is actually a directory - if params.file.IsDirectory() { - return addDir(n, params) + if file.IsDirectory() { + return params.addDir(file, ignoreFilePatterns) } // if the progress flag was specified, wrap the file so that we can send // progress updates to the client (over the output channel) - var reader io.Reader = params.file + var reader io.Reader = file if params.progress { - reader = &progressReader{file: params.file, out: params.out} + reader = &progressReader{file: file, out: params.out} } if params.wrap { - p, dagnode, err := coreunix.AddWrapped(n, reader, path.Base(params.file.FileName())) + p, dagnode, err := coreunix.AddWrapped(params.node, reader, path.Base(file.FileName())) if err != nil { return nil, err } params.out <- &AddedObject{ Hash: p, - Name: params.file.FileName(), + Name: file.FileName(), } return dagnode, nil } - dagnode, err := add(n, reader) + dagnode, err := add(params.node, reader) if err != nil { return nil, err } - log.Infof("adding file: %s", params.file.FileName()) - if err := outputDagnode(params.out, params.file.FileName(), dagnode); err != nil { + log.Infof("adding file: %s", file.FileName()) + if err := outputDagnode(params.out, file.FileName(), dagnode); err != nil { return nil, err } return dagnode, nil } -func addDir(n *core.IpfsNode, params addParams) (*dag.Node, error) { +func (params *adder) addDir(file files.File, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { tree := &dag.Node{Data: ft.FolderPBData()} - log.Infof("adding directory: %s", params.file.FileName()) + log.Infof("adding directory: %s", file.FileName()) // Check for an .ipfsignore file that is local to this Dir and append to the incoming - localIgnorePatterns := checkForLocalIgnorePatterns(params.file.FileName(), params.ignoreFilePatterns) + localIgnorePatterns := checkForLocalIgnorePatterns(file.FileName(), ignoreFilePatterns) for { - file, err := params.file.NextFile() + file, err := file.NextFile() if err != nil && err != io.EOF { return nil, err } @@ -326,7 +326,7 @@ func addDir(n *core.IpfsNode, params addParams) (*dag.Node, error) { break } - node, err := addFile(n, addParams{file, params.out, params.progress, false, params.hidden, localIgnorePatterns}) + node, err := params.addFile(file, localIgnorePatterns) if _, ok := err.(*hiddenFileError); ok { // hidden file error, set the node to nil for below node = nil @@ -347,12 +347,12 @@ func addDir(n *core.IpfsNode, params addParams) (*dag.Node, error) { } } - err := outputDagnode(params.out, params.file.FileName(), tree) + err := outputDagnode(params.out, file.FileName(), tree) if err != nil { return nil, err } - _, err = n.DAG.Add(tree) + _, err = params.node.DAG.Add(tree) if err != nil { return nil, err } From f04ac6f274950a9a86a9957fd899a7f06a50aea6 Mon Sep 17 00:00:00 2001 From: gatesvp Date: Wed, 13 May 2015 22:19:31 -0700 Subject: [PATCH 07/22] Add recursive check for parent .ipfsignore files --- core/commands/add.go | 45 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 46c06c76f42..170c3cc06d4 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "path" + "path/filepath" "strings" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb" @@ -123,11 +124,7 @@ remains to be implemented. // If the file is not a folder, then let's get the root of that // folder and attempt to load the appropriate .ipfsignore. - localIgnorePatterns := ignoreFilePatterns - if !file.IsDirectory() { - parentPath := path.Dir(file.FileName()) - localIgnorePatterns = checkForLocalIgnorePatterns(parentPath, ignoreFilePatterns) - } + localIgnorePatterns := checkForParentIgnorePatterns(file.FileName(), ignoreFilePatterns) addParams := adder{n, outChan, progress, wrap, hidden} rootnd, err := addParams.addFile(file, localIgnorePatterns) @@ -236,6 +233,7 @@ remains to be implemented. Type: AddedObject{}, } +// Internal structure for holding the switches passed to the `add` call type adder struct { node *core.IpfsNode out chan interface{} @@ -244,6 +242,7 @@ type adder struct { hidden bool } +// Perform the actual add & pin locally, outputting results to reader func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { node, err := importer.BuildDagFromReader(reader, n.DAG, nil, chunk.DefaultSplitter) if err != nil { @@ -258,6 +257,9 @@ func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { return node, nil } +// Add the given file while respecting the params and ignoreFilePatterns. +// Note that ignoreFilePatterns is not part of the struct as it may change while +// we dig through folders. func (params *adder) addFile(file files.File, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { // Check if file is hidden if fileIsHidden := files.IsHidden(file); fileIsHidden && !params.hidden { @@ -360,7 +362,10 @@ func (params *adder) addDir(file files.File, ignoreFilePatterns []ignore.GitIgno return tree, nil } +// this helper checks the local path for any .ipfsignore file that need to be +// respected. returns the updated or the original GitIgnore. func checkForLocalIgnorePatterns(dir string, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore { + ignorePathname := path.Join(dir, ".ipfsignore") localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname) @@ -372,6 +377,36 @@ func checkForLocalIgnorePatterns(dir string, ignoreFilePatterns []ignore.GitIgno } } +// this helper just walks the parent directories of the given path looking for +// any .ipfsignore files in those directories. +func checkForParentIgnorePatterns(givenPath string, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore { + absolutePath, err := filepath.Abs(givenPath) + + if err != nil { + return ignoreFilePatterns + } + + // break out the absolute path + dir := filepath.Dir(absolutePath) + pathComponents := strings.Split(dir, "\\") + + // We loop through each parent component attempting to find an .ipfsignore file + for index, _ := range pathComponents { + + pathParts := make([]string, len(pathComponents)+1) + copy(pathParts, pathComponents[0:index+1]) + ignorePathname := path.Join(append(pathParts, ".ipfsignore")...) + + localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname) + if ignoreErr == nil && localIgnore != nil { + log.Debugf("found ignore file: %s", dir) + ignoreFilePatterns = append(ignoreFilePatterns, *localIgnore) + } + } + + return ignoreFilePatterns +} + // outputDagnode sends dagnode info over the output channel func outputDagnode(out chan interface{}, name string, dn *dag.Node) error { o, err := getOutput(dn) From 7f818e223b8a2077b51167eb936e87f82ef1745c Mon Sep 17 00:00:00 2001 From: gatesvp Date: Wed, 13 May 2015 23:02:37 -0700 Subject: [PATCH 08/22] Respect file path separator on all OSes --- core/commands/add.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/commands/add.go b/core/commands/add.go index 170c3cc06d4..151cf979caa 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -5,6 +5,7 @@ import ( "io" "path" "path/filepath" + "strconv" "strings" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb" @@ -388,7 +389,7 @@ func checkForParentIgnorePatterns(givenPath string, ignoreFilePatterns []ignore. // break out the absolute path dir := filepath.Dir(absolutePath) - pathComponents := strings.Split(dir, "\\") + pathComponents := strings.Split(dir, strconv.QuoteRune(filepath.Separator)) // We loop through each parent component attempting to find an .ipfsignore file for index, _ := range pathComponents { From 032cd3cf9b31d997b1752600c3f0b604638640f2 Mon Sep 17 00:00:00 2001 From: gatesvp Date: Thu, 14 May 2015 02:54:28 -0400 Subject: [PATCH 09/22] Fix is_hidden on non-Windows --- commands/files/is_hidden.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/files/is_hidden.go b/commands/files/is_hidden.go index 0aeb6ba516e..b0360685b0d 100644 --- a/commands/files/is_hidden.go +++ b/commands/files/is_hidden.go @@ -12,8 +12,8 @@ func IsHidden(f File) bool { fName := filepath.Base(f.FileName()) if strings.HasPrefix(fName, ".") && len(fName) > 1 { - return true, nil + return true } - return false, nil + return false } From 70dc7d977e011511d983ce42a2e417ecdac8031b Mon Sep 17 00:00:00 2001 From: gatesvp Date: Thu, 14 May 2015 03:16:57 -0400 Subject: [PATCH 10/22] Vendoring for .ipfsignore support --- Godeps/Godeps.json | 4 + .../sabhiram/go-git-ignore/.gitignore | 28 ++ .../sabhiram/go-git-ignore/.travis.yml | 18 ++ .../github.com/sabhiram/go-git-ignore/LICENSE | 22 ++ .../sabhiram/go-git-ignore/README.md | 17 ++ .../sabhiram/go-git-ignore/ignore.go | 172 ++++++++++++ .../sabhiram/go-git-ignore/ignore_test.go | 259 ++++++++++++++++++ core/commands/add.go | 3 +- 8 files changed, 521 insertions(+), 2 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.gitignore create mode 100644 Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.travis.yml create mode 100644 Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/LICENSE create mode 100644 Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/README.md create mode 100644 Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore.go create mode 100644 Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 1ab026f0257..a338c2597e8 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -237,6 +237,10 @@ "ImportPath": "github.com/rs/cors", "Rev": "5e4ce6bc0ecd3472f6f943666d84876691be2ced" }, + { + "ImportPath": "github.com/sabhiram/go-git-ignore", + "Rev": "f9a1328f5fc50414f8751f587774ccd3f49b492b" + }, { "ImportPath": "github.com/steakknife/hamming", "Comment": "0.0.10", diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.gitignore b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.gitignore new file mode 100644 index 00000000000..0e919aff1c2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.gitignore @@ -0,0 +1,28 @@ +# Package test fixtures +test_fixtures + +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.travis.yml b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.travis.yml new file mode 100644 index 00000000000..24ddadf1bf6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.travis.yml @@ -0,0 +1,18 @@ +language: go + +go: + - 1.3 + - tip + +env: + - "PATH=$HOME/gopath/bin:$PATH" + +before_install: + - go get github.com/stretchr/testify/assert + - go get github.com/axw/gocov/gocov + - go get github.com/mattn/goveralls + - if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi + +script: + - go test -v -covermode=count -coverprofile=coverage.out + - goveralls -coverprofile=coverage.out -service travis-ci -repotoken $COVERALLS_TOKEN diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/LICENSE b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/LICENSE new file mode 100644 index 00000000000..c606f49e5c0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Shaba Abhiram + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/README.md b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/README.md new file mode 100644 index 00000000000..fbbb3761dc0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/README.md @@ -0,0 +1,17 @@ +# go-git-ignore + +[![Build Status](https://travis-ci.org/sabhiram/go-git-ignore.svg)](https://travis-ci.org/sabhiram/go-git-ignore) [![Coverage Status](https://coveralls.io/repos/sabhiram/go-git-ignore/badge.png?branch=master)](https://coveralls.io/r/sabhiram/go-git-ignore?branch=master) + +A gitignore parser for `Go` + +## Install + +```shell +go get github.com/sabhiram/go-git-ignore +``` + +## Usage + +```shell +TODO +``` diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore.go b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore.go new file mode 100644 index 00000000000..b15f0d8f2e7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore.go @@ -0,0 +1,172 @@ +/* +ignore is a library which returns a new ignorer object which can +test against various paths. This is particularly useful when trying +to filter files based on a .gitignore document + +The rules for parsing the input file are the same as the ones listed +in the Git docs here: http://git-scm.com/docs/gitignore + +The summarized version of the same has been copied here: + + 1. A blank line matches no files, so it can serve as a separator + for readability. + 2. A line starting with # serves as a comment. Put a backslash ("\") + in front of the first hash for patterns that begin with a hash. + 3. Trailing spaces are ignored unless they are quoted with backslash ("\"). + 4. An optional prefix "!" which negates the pattern; any matching file + excluded by a previous pattern will become included again. It is not + possible to re-include a file if a parent directory of that file is + excluded. Git doesn’t list excluded directories for performance reasons, + so any patterns on contained files have no effect, no matter where they + are defined. Put a backslash ("\") in front of the first "!" for + patterns that begin with a literal "!", for example, "\!important!.txt". + 5. If the pattern ends with a slash, it is removed for the purpose of the + following description, but it would only find a match with a directory. + In other words, foo/ will match a directory foo and paths underneath it, + but will not match a regular file or a symbolic link foo (this is + consistent with the way how pathspec works in general in Git). + 6. If the pattern does not contain a slash /, Git treats it as a shell glob + pattern and checks for a match against the pathname relative to the + location of the .gitignore file (relative to the toplevel of the work + tree if not from a .gitignore file). + 7. Otherwise, Git treats the pattern as a shell glob suitable for + consumption by fnmatch(3) with the FNM_PATHNAME flag: wildcards in the + pattern will not match a / in the pathname. For example, + "Documentation/*.html" matches "Documentation/git.html" but not + "Documentation/ppc/ppc.html" or "tools/perf/Documentation/perf.html". + 8. A leading slash matches the beginning of the pathname. For example, + "/*.c" matches "cat-file.c" but not "mozilla-sha1/sha1.c". + 9. Two consecutive asterisks ("**") in patterns matched against full + pathname may have special meaning: + i. A leading "**" followed by a slash means match in all directories. + For example, "** /foo" matches file or directory "foo" anywhere, + the same as pattern "foo". "** /foo/bar" matches file or directory + "bar" anywhere that is directly under directory "foo". + ii. A trailing "/**" matches everything inside. For example, "abc/**" + matches all files inside directory "abc", relative to the location + of the .gitignore file, with infinite depth. + iii. A slash followed by two consecutive asterisks then a slash matches + zero or more directories. For example, "a/** /b" matches "a/b", + "a/x/b", "a/x/y/b" and so on. + iv. Other consecutive asterisks are considered invalid. */ +package ignore + +import ( + "strings" + "regexp" + "io/ioutil" +) + +// An IgnoreParser is an interface which exposes two methods: +// MatchesPath() - Returns true if the path is targeted by the patterns compiled in the GitIgnore structure +type IgnoreParser interface { + IncludesPath(f string) bool + IgnoresPath(f string) bool + MatchesPath(f string) bool +} + +// GitIgnore is a struct which contains a slice of regexp.Regexp +// patterns +type GitIgnore struct { + patterns []*regexp.Regexp // List of regexp patterns which this ignore file applies + negate []bool // List of booleans which determine if the pattern is negated +} + +// This function pretty much attempts to mimic the parsing rules +// listed above at the start of this file +func getPatternFromLine(line string) (*regexp.Regexp, bool) { + // Strip comments [Rule 2] + if regexp.MustCompile(`^#`).MatchString(line) { return nil, false } + + // Trim string [Rule 3] + // TODO: Hanlde [Rule 3], when the " " is escaped with a \ + line = strings.Trim(line, " ") + + // Exit for no-ops and return nil which will prevent us from + // appending a pattern against this line + if line == "" { return nil, false } + + // TODO: Handle [Rule 4] which negates the match for patterns leading with "!" + negatePattern := false + if string(line[0]) == "!" { + negatePattern = true + line = line[1:] + } + + // Handle [Rule 2, 4], when # or ! is escaped with a \ + // Handle [Rule 4] once we tag negatePattern, strip the leading ! char + if regexp.MustCompile(`^(\#|\!)`).MatchString(line) { + line = line[1:] + } + + // Handle [Rule 8], strip leading / and enforce path checking if its present + if regexp.MustCompile(`^/`).MatchString(line) { + line = "^" + line[1:] + } + + // If we encounter a foo/*.blah in a folder, prepend the ^ char + if regexp.MustCompile(`([^\/+])/.*\*\.`).MatchString(line) { + line = "^" + line + } + + // Handle escaping the "." char + line = regexp.MustCompile(`\.`).ReplaceAllString(line, `\.`) + + // Handle "**" usage (and special case when it is followed by a /) + line = regexp.MustCompile(`\*\*(/|)`).ReplaceAllString(line, `(.+|)`) + + // Handle escaping the "*" char + line = regexp.MustCompile(`\*`).ReplaceAllString(line, `([^\/]+)`) + + + // Temporary regex + expr := line + "(|/.+)$" + pattern, _ := regexp.Compile(expr) + + return pattern, negatePattern +} + +// Accepts a variadic set of strings, and returns a GitIgnore object which +// converts and appends the lines in the input to regexp.Regexp patterns +// held within the GitIgnore objects "patterns" field +func CompileIgnoreLines(lines ...string) (*GitIgnore, error) { + g := new(GitIgnore) + for _, line := range lines { + pattern, negatePattern := getPatternFromLine(line) + if pattern != nil { + g.patterns = append(g.patterns, pattern) + g.negate = append(g.negate, negatePattern) + } + } + return g, nil +} + +// Accepts a ignore file as the input, parses the lines out of the file +// and invokes the CompileIgnoreLines method +func CompileIgnoreFile(fpath string) (*GitIgnore, error) { + buffer, error := ioutil.ReadFile(fpath) + if error == nil { + s := strings.Split(string(buffer), "\n") + return CompileIgnoreLines(s...) + } + return nil, error +} + +// MatchesPath is an interface function for the IgnoreParser interface. +// It returns true if the given GitIgnore structure would target a given +// path string "f" +func (g GitIgnore) MatchesPath(f string) bool { + matchesPath := false + for idx, pattern := range g.patterns { + if pattern.MatchString(f) { + // If this is a regular target (not negated with a gitignore exclude "!" etc) + if !g.negate[idx] { + matchesPath = true + // Negated pattern, and matchesPath is already set + } else if matchesPath { + matchesPath = false + } + } + } + return matchesPath +} diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore_test.go b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore_test.go new file mode 100644 index 00000000000..3893f517d65 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore_test.go @@ -0,0 +1,259 @@ +// Implement tests for the `ignore` library +package ignore + +import ( + "os" + + "io/ioutil" + "path/filepath" + + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + TEST_DIR = "test_fixtures" +) + +// Helper function to setup a test fixture dir and write to +// it a file with the name "fname" and content "content" +func writeFileToTestDir(fname, content string) { + testDirPath := "." + string(filepath.Separator) + TEST_DIR + testFilePath := testDirPath + string(filepath.Separator) + fname + + _ = os.MkdirAll(testDirPath, 0755) + _ = ioutil.WriteFile(testFilePath, []byte(content), os.ModePerm) +} + +func cleanupTestDir() { + _ = os.RemoveAll(fmt.Sprintf(".%s%s", string(filepath.Separator), TEST_DIR)) +} + +// Validate "CompileIgnoreLines()" +func TestCompileIgnoreLines(test *testing.T) { + lines := []string{"abc/def", "a/b/c", "b"} + object, error := CompileIgnoreLines(lines...) + assert.Nil(test, error, "error from CompileIgnoreLines should be nil") + + // MatchesPath + // Paths which are targeted by the above "lines" + assert.Equal(test, true, object.MatchesPath("abc/def/child"), "abc/def/child should match") + assert.Equal(test, true, object.MatchesPath("a/b/c/d"), "a/b/c/d should match") + + // Paths which are not targeted by the above "lines" + assert.Equal(test, false, object.MatchesPath("abc"), "abc should not match") + assert.Equal(test, false, object.MatchesPath("def"), "def should not match") + assert.Equal(test, false, object.MatchesPath("bd"), "bd should not match") + + object, error = CompileIgnoreLines("abc/def", "a/b/c", "b") + assert.Nil(test, error, "error from CompileIgnoreLines should be nil") + + // Paths which are targeted by the above "lines" + assert.Equal(test, true, object.MatchesPath("abc/def/child"), "abc/def/child should match") + assert.Equal(test, true, object.MatchesPath("a/b/c/d"), "a/b/c/d should match") + + // Paths which are not targeted by the above "lines" + assert.Equal(test, false, object.MatchesPath("abc"), "abc should not match") + assert.Equal(test, false, object.MatchesPath("def"), "def should not match") + assert.Equal(test, false, object.MatchesPath("bd"), "bd should not match") +} + +// Validate the invalid files +func TestCompileIgnoreFile_InvalidFile(test *testing.T) { + object, error := CompileIgnoreFile("./test_fixtures/invalid.file") + assert.Nil(test, object, "object should be nil") + assert.NotNil(test, error, "error should be unknown file / dir") +} + +// Validate the an empty files +func TestCompileIgnoreLines_EmptyFile(test *testing.T) { + writeFileToTestDir("test.gitignore", ``) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, false, object.MatchesPath("a"), "should not match any path") + assert.Equal(test, false, object.MatchesPath("a/b"), "should not match any path") + assert.Equal(test, false, object.MatchesPath(".foobar"), "should not match any path") +} + +// Validate the correct handling of the negation operator "!" +func TestCompileIgnoreLines_HandleIncludePattern(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +# exclude everything except directory foo/bar +/* +!/foo +/foo/* +!/foo/bar +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("a"), "a should match") + assert.Equal(test, true, object.MatchesPath("foo/baz"), "foo/baz should match") + assert.Equal(test, false, object.MatchesPath("foo"), "foo should not match") + assert.Equal(test, false, object.MatchesPath("/foo/bar"), "/foo/bar should not match") +} + +// Validate the correct handling of comments and empty lines +func TestCompileIgnoreLines_HandleSpaces(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +# +# A comment + +# Another comment + + + # Invalid Comment + +abc/def +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, 2, len(object.patterns), "should have two regex pattern") + assert.Equal(test, false, object.MatchesPath("abc/abc"), "/abc/abc should not match") + assert.Equal(test, true, object.MatchesPath("abc/def"), "/abc/def should match") +} + +// Validate the correct handling of leading / chars +func TestCompileIgnoreLines_HandleLeadingSlash(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +/a/b/c +d/e/f +/g +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, 3, len(object.patterns), "should have 3 regex patterns") + assert.Equal(test, true, object.MatchesPath("a/b/c"), "a/b/c should match") + assert.Equal(test, true, object.MatchesPath("a/b/c/d"), "a/b/c/d should match") + assert.Equal(test, true, object.MatchesPath("d/e/f"), "d/e/f should match") + assert.Equal(test, true, object.MatchesPath("g"), "g should match") +} + +// Validate the correct handling of files starting with # or ! +func TestCompileIgnoreLines_HandleLeadingSpecialChars(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +# Comment +\#file.txt +\!file.txt +file.txt +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("#file.txt"), "#file.txt should match") + assert.Equal(test, true, object.MatchesPath("!file.txt"), "!file.txt should match") + assert.Equal(test, true, object.MatchesPath("a/!file.txt"), "a/!file.txt should match") + assert.Equal(test, true, object.MatchesPath("file.txt"), "file.txt should match") + assert.Equal(test, true, object.MatchesPath("a/file.txt"), "a/file.txt should match") + assert.Equal(test, false, object.MatchesPath("file2.txt"), "file2.txt should not match") + +} + +// Validate the correct handling matching files only within a given folder +func TestCompileIgnoreLines_HandleAllFilesInDir(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +Documentation/*.html +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("Documentation/git.html"), "Documentation/git.html should match") + assert.Equal(test, false, object.MatchesPath("Documentation/ppc/ppc.html"), "Documentation/ppc/ppc.html should not match") + assert.Equal(test, false, object.MatchesPath("tools/perf/Documentation/perf.html"), "tools/perf/Documentation/perf.html should not match") +} + +// Validate the correct handling of "**" +func TestCompileIgnoreLines_HandleDoubleStar(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +**/foo +bar +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("foo"), "foo should match") + assert.Equal(test, true, object.MatchesPath("baz/foo"), "baz/foo should match") + assert.Equal(test, true, object.MatchesPath("bar"), "bar should match") + assert.Equal(test, true, object.MatchesPath("baz/bar"), "baz/bar should match") +} + +// Validate the correct handling of leading slash +func TestCompileIgnoreLines_HandleLeadingSlashPath(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +/*.c +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("hello.c"), "hello.c should match") + assert.Equal(test, false, object.MatchesPath("foo/hello.c"), "foo/hello.c should not match") +} + +func ExampleCompileIgnoreLines() { + ignoreObject, error := CompileIgnoreLines([]string{"node_modules", "*.out", "foo/*.c"}...) + if error != nil { + panic("Error when compiling ignore lines: " + error.Error()) + } + + // You can test the ignoreObject against various paths using the + // "MatchesPath()" interface method. This pretty much is up to + // the users interpretation. In the case of a ".gitignore" file, + // a "match" would indicate that a given path would be ignored. + fmt.Println(ignoreObject.MatchesPath("node_modules/test/foo.js")) + fmt.Println(ignoreObject.MatchesPath("node_modules2/test.out")) + fmt.Println(ignoreObject.MatchesPath("test/foo.js")) + + // Output: + // true + // true + // false +} + +func TestCompileIgnoreLines_CheckNestedDotFiles(test *testing.T) { + lines := []string{ + "**/external/**/*.md", + "**/external/**/*.json", + "**/external/**/*.gzip", + "**/external/**/.*ignore", + + "**/external/foobar/*.css", + "**/external/barfoo/less", + "**/external/barfoo/scss", + } + object, error := CompileIgnoreLines(lines...) + assert.Nil(test, error, "error from CompileIgnoreLines should be nil") + assert.NotNil(test, object, "returned object should not be nil") + + assert.Equal(test, true, object.MatchesPath("external/foobar/angular.foo.css"), "external/foobar/angular.foo.css") + assert.Equal(test, true, object.MatchesPath("external/barfoo/.gitignore"), "external/barfoo/.gitignore") + assert.Equal(test, true, object.MatchesPath("external/barfoo/.bower.json"), "external/barfoo/.bower.json") +} diff --git a/core/commands/add.go b/core/commands/add.go index 151cf979caa..89293df01a6 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -9,9 +9,8 @@ import ( "strings" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb" + ignore "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" - ignore "github.com/sabhiram/go-git-ignore" - cmds "github.com/ipfs/go-ipfs/commands" files "github.com/ipfs/go-ipfs/commands/files" core "github.com/ipfs/go-ipfs/core" From 8e9a4778ceb1aca19dd43d556eba32d3628d1ba8 Mon Sep 17 00:00:00 2001 From: gatesvp Date: Thu, 14 May 2015 00:52:18 -0700 Subject: [PATCH 11/22] Fix OS-specific slashing without strconv --- core/commands/add.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 151cf979caa..72cd4e619c2 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -5,7 +5,6 @@ import ( "io" "path" "path/filepath" - "strconv" "strings" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb" @@ -389,7 +388,7 @@ func checkForParentIgnorePatterns(givenPath string, ignoreFilePatterns []ignore. // break out the absolute path dir := filepath.Dir(absolutePath) - pathComponents := strings.Split(dir, strconv.QuoteRune(filepath.Separator)) + pathComponents := strings.Split(dir, string(filepath.Separator)) // We loop through each parent component attempting to find an .ipfsignore file for index, _ := range pathComponents { @@ -400,7 +399,7 @@ func checkForParentIgnorePatterns(givenPath string, ignoreFilePatterns []ignore. localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname) if ignoreErr == nil && localIgnore != nil { - log.Debugf("found ignore file: %s", dir) + log.Debugf("found ignore file: %s", ignorePathname) ignoreFilePatterns = append(ignoreFilePatterns, *localIgnore) } } From 9e5dfe2fc99c3c256b21887eb74fbc7a9b687055 Mon Sep 17 00:00:00 2001 From: gatesvp Date: Sun, 3 May 2015 22:59:27 -0700 Subject: [PATCH 12/22] Make ipfs add ignore files by default. --- commands/files/is_hidden.go | 19 +++++++++ commands/files/is_hidden_windows.go | 29 +++++++++++++ commands/files/serialfile.go | 2 +- core/commands/add.go | 66 +++++++++++++++++------------ 4 files changed, 89 insertions(+), 27 deletions(-) create mode 100644 commands/files/is_hidden.go create mode 100644 commands/files/is_hidden_windows.go diff --git a/commands/files/is_hidden.go b/commands/files/is_hidden.go new file mode 100644 index 00000000000..004ec04e51a --- /dev/null +++ b/commands/files/is_hidden.go @@ -0,0 +1,19 @@ +// +build !windows + +package files + +import ( + "path/filepath" + "strings" +) + +func IsHidden(f File) bool { + + fName := filepath.Base(f.FileName()) + + if strings.HasPrefix(fName, ".") { + return true, nil + } + + return false, nil +} diff --git a/commands/files/is_hidden_windows.go b/commands/files/is_hidden_windows.go new file mode 100644 index 00000000000..2f310d4448a --- /dev/null +++ b/commands/files/is_hidden_windows.go @@ -0,0 +1,29 @@ +// +build windows + +package files + +import ( + "path/filepath" + "strings" + "syscall" +) + +func IsHidden(f File) bool { + + fName := filepath.Base(f.FileName()) + + if strings.HasPrefix(fName, ".") { + return true + } + + p, e := syscall.UTF16PtrFromString(f.FileName()) + if e != nil { + return false + } + + attrs, e := syscall.GetFileAttributes(p) + if e != nil { + return false + } + return attrs&syscall.FILE_ATTRIBUTE_HIDDEN != 0, nil +} diff --git a/commands/files/serialfile.go b/commands/files/serialfile.go index aeba01fa7ed..461bde33600 100644 --- a/commands/files/serialfile.go +++ b/commands/files/serialfile.go @@ -3,7 +3,7 @@ package files import ( "io" "os" - fp "path" + fp "path/filepath" "sort" "syscall" ) diff --git a/core/commands/add.go b/core/commands/add.go index 85692337b7f..501b94a0820 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -29,6 +29,7 @@ const progressReaderIncrement = 1024 * 256 const ( progressOptionName = "progress" wrapOptionName = "wrap-with-directory" + hiddenOptionName = "hidden" ) type AddedObject struct { @@ -57,6 +58,7 @@ remains to be implemented. cmds.BoolOption(progressOptionName, "p", "Stream progress data"), cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object"), cmds.BoolOption("t", "trickle", "Use trickle-dag format for dag generation"), + cmds.BoolOption(hiddenOptionName, "Include files that are hidden"), }, PreRun: func(req cmds.Request) error { if quiet, _, _ := req.Option("quiet").Bool(); quiet { @@ -90,6 +92,7 @@ remains to be implemented. progress, _, _ := req.Option(progressOptionName).Bool() wrap, _, _ := req.Option(wrapOptionName).Bool() + hidden, _, _ := req.Option(hiddenOptionName).Bool() outChan := make(chan interface{}) res.SetOutput((<-chan interface{})(outChan)) @@ -107,7 +110,7 @@ remains to be implemented. return } - rootnd, err := addFile(n, file, outChan, progress, wrap) + rootnd, err := addFile(n, file, outChan, progress, wrap, hidden) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -227,9 +230,12 @@ func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { return node, nil } -func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool, wrap bool) (*dag.Node, error) { +func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool, wrap bool, hidden bool) (*dag.Node, error) { if file.IsDirectory() { - return addDir(n, file, out, progress) + return addDir(n, file, out, progress, hidden) + } else if fileIsHidden := files.IsHidden(file); fileIsHidden && !hidden { + log.Infof("%s is a hidden file, skipping", file.FileName()) + return nil, &hiddenFileError{file.FileName()} } // if the progress flag was specified, wrap the file so that we can send @@ -263,43 +269,51 @@ func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress b return dagnode, nil } -func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool) (*dag.Node, error) { - log.Infof("adding directory: %s", dir.FileName()) +func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool, hidden bool) (*dag.Node, error) { tree := &dag.Node{Data: ft.FolderPBData()} + if dirIsHidden := files.IsHidden(dir); dirIsHidden && !hidden { + log.Infof("ignoring directory: %s", dir.FileName()) + } else { + log.Infof("adding directory: %s", dir.FileName()) + for { + file, err := dir.NextFile() + if err != nil && err != io.EOF { + return nil, err + } + if file == nil { + break + } - for { - file, err := dir.NextFile() - if err != nil && err != io.EOF { - return nil, err - } - if file == nil { - break + node, err := addFile(n, file, out, progress, false, hidden) + if _, ok := err.(*hiddenFileError); ok { + // hidden file error, set the node to nil for below + node = nil + } else if err != nil { + return nil, err + } + + if node != nil { + _, name := path.Split(file.FileName()) + + err = tree.AddNodeLink(name, node) + if err != nil { + return nil, err + } + } } - node, err := addFile(n, file, out, progress, false) + err := outputDagnode(out, dir.FileName(), tree) if err != nil { return nil, err } - _, name := path.Split(file.FileName()) - - err = tree.AddNodeLink(name, node) + _, err = n.DAG.Add(tree) if err != nil { return nil, err } } - err := outputDagnode(out, dir.FileName(), tree) - if err != nil { - return nil, err - } - - _, err = n.DAG.Add(tree) - if err != nil { - return nil, err - } - return tree, nil } From 1914a2690fb656b94d8d52cb378cc66d07a1e7aa Mon Sep 17 00:00:00 2001 From: gatesvp Date: Sun, 3 May 2015 23:10:24 -0700 Subject: [PATCH 13/22] Fix my build --- commands/files/is_hidden_windows.go | 2 +- core/commands/add.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/commands/files/is_hidden_windows.go b/commands/files/is_hidden_windows.go index 2f310d4448a..844610fb8ec 100644 --- a/commands/files/is_hidden_windows.go +++ b/commands/files/is_hidden_windows.go @@ -25,5 +25,5 @@ func IsHidden(f File) bool { if e != nil { return false } - return attrs&syscall.FILE_ATTRIBUTE_HIDDEN != 0, nil + return attrs&syscall.FILE_ATTRIBUTE_HIDDEN != 0 } diff --git a/core/commands/add.go b/core/commands/add.go index 501b94a0820..3bad0983267 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -332,6 +332,14 @@ func outputDagnode(out chan interface{}, name string, dn *dag.Node) error { return nil } +type hiddenFileError struct { + fileName string +} + +func (e *hiddenFileError) Error() string { + return fmt.Sprintf("%s is a hidden file", e.fileName) +} + type progressReader struct { file files.File out chan interface{} From 153886ea4244daae578ae24fc8e6b856aec8e040 Mon Sep 17 00:00:00 2001 From: gatesvp Date: Wed, 6 May 2015 00:33:17 -0700 Subject: [PATCH 14/22] ipfs add now respects .ipfsignore files --- commands/files/is_hidden.go | 2 +- commands/files/is_hidden_windows.go | 2 +- core/commands/add.go | 130 +++++++++++++++++++--------- 3 files changed, 93 insertions(+), 41 deletions(-) diff --git a/commands/files/is_hidden.go b/commands/files/is_hidden.go index 004ec04e51a..0aeb6ba516e 100644 --- a/commands/files/is_hidden.go +++ b/commands/files/is_hidden.go @@ -11,7 +11,7 @@ func IsHidden(f File) bool { fName := filepath.Base(f.FileName()) - if strings.HasPrefix(fName, ".") { + if strings.HasPrefix(fName, ".") && len(fName) > 1 { return true, nil } diff --git a/commands/files/is_hidden_windows.go b/commands/files/is_hidden_windows.go index 844610fb8ec..5d263931033 100644 --- a/commands/files/is_hidden_windows.go +++ b/commands/files/is_hidden_windows.go @@ -12,7 +12,7 @@ func IsHidden(f File) bool { fName := filepath.Base(f.FileName()) - if strings.HasPrefix(fName, ".") { + if strings.HasPrefix(fName, ".") && len(fName) > 1 { return true } diff --git a/core/commands/add.go b/core/commands/add.go index 3bad0983267..46ce7dbcab2 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -3,11 +3,13 @@ package commands import ( "fmt" "io" + "os" "path" "strings" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + ignore "github.com/sabhiram/go-git-ignore" cmds "github.com/ipfs/go-ipfs/commands" files "github.com/ipfs/go-ipfs/commands/files" @@ -94,6 +96,16 @@ remains to be implemented. wrap, _, _ := req.Option(wrapOptionName).Bool() hidden, _, _ := req.Option(hiddenOptionName).Bool() + var ignoreFilePatterns []ignore.GitIgnore + + // Check the $IPFS_PATH + if ipfs_path := os.Getenv("$IPFS_PATH"); len(ipfs_path) > 0 { + baseFilePattern, err := ignore.CompileIgnoreFile(path.Join(ipfs_path, ".ipfsignore")) + if err == nil && baseFilePattern != nil { + ignoreFilePatterns = append(ignoreFilePatterns, *baseFilePattern) + } + } + outChan := make(chan interface{}) res.SetOutput((<-chan interface{})(outChan)) @@ -110,7 +122,7 @@ remains to be implemented. return } - rootnd, err := addFile(n, file, outChan, progress, wrap, hidden) + rootnd, err := addFile(n, file, outChan, progress, wrap, hidden, ignoreFilePatterns) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -230,14 +242,26 @@ func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { return node, nil } -func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool, wrap bool, hidden bool) (*dag.Node, error) { - if file.IsDirectory() { - return addDir(n, file, out, progress, hidden) - } else if fileIsHidden := files.IsHidden(file); fileIsHidden && !hidden { - log.Infof("%s is a hidden file, skipping", file.FileName()) +func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool, wrap bool, hidden bool, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { + // Check if file is hidden + if fileIsHidden := files.IsHidden(file); fileIsHidden && !hidden { + log.Debugf("%s is hidden, skipping", file.FileName()) return nil, &hiddenFileError{file.FileName()} } + // Check for ignore files matches + for i := range ignoreFilePatterns { + if ignoreFilePatterns[i].MatchesPath(file.FileName()) { + log.Debugf("%s is ignored file, skipping", file.FileName()) + return nil, &ignoreFileError{file.FileName()} + } + } + + // Check if "file" is actually a directory + if file.IsDirectory() { + return addDir(n, file, out, progress, hidden, ignoreFilePatterns) + } + // if the progress flag was specified, wrap the file so that we can send // progress updates to the client (over the output channel) var reader io.Reader = file @@ -269,54 +293,74 @@ func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress b return dagnode, nil } -func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool, hidden bool) (*dag.Node, error) { +func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool, hidden bool, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { tree := &dag.Node{Data: ft.FolderPBData()} - if dirIsHidden := files.IsHidden(dir); dirIsHidden && !hidden { - log.Infof("ignoring directory: %s", dir.FileName()) - } else { - log.Infof("adding directory: %s", dir.FileName()) - for { - file, err := dir.NextFile() - if err != nil && err != io.EOF { - return nil, err - } - if file == nil { - break - } - - node, err := addFile(n, file, out, progress, false, hidden) - if _, ok := err.(*hiddenFileError); ok { - // hidden file error, set the node to nil for below - node = nil - } else if err != nil { - return nil, err - } + log.Infof("adding directory: %s", dir.FileName()) - if node != nil { - _, name := path.Split(file.FileName()) + // Check for an .ipfsignore file that is local to this Dir and append to the incoming + localIgnorePatterns := checkForLocalIgnorePatterns(dir, ignoreFilePatterns) - err = tree.AddNodeLink(name, node) - if err != nil { - return nil, err - } - } + for { + file, err := dir.NextFile() + if err != nil && err != io.EOF { + return nil, err + } + if file == nil { + break } - err := outputDagnode(out, dir.FileName(), tree) - if err != nil { + node, err := addFile(n, file, out, progress, false, hidden, localIgnorePatterns) + if _, ok := err.(*hiddenFileError); ok { + // hidden file error, set the node to nil for below + node = nil + } else if _, ok := err.(*ignoreFileError); ok { + // ignore file error, set the node to nil for below + node = nil + } else if err != nil { return nil, err } - _, err = n.DAG.Add(tree) - if err != nil { - return nil, err + if node != nil { + _, name := path.Split(file.FileName()) + + err = tree.AddNodeLink(name, node) + if err != nil { + return nil, err + } } } + err := outputDagnode(out, dir.FileName(), tree) + if err != nil { + return nil, err + } + + _, err = n.DAG.Add(tree) + if err != nil { + return nil, err + } + return tree, nil } +func checkForLocalIgnorePatterns(dir files.File, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore { + var ignorePathname string + if dir.FileName() == "." { + ignorePathname = ".ipfsignore" + } else { + ignorePathname = path.Join(dir.FileName(), ".ipfsignore") + } + + localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname) + if ignoreErr == nil && localIgnore != nil { + log.Debugf("found ignore file: %s", dir.FileName()) + return append(ignoreFilePatterns, *localIgnore) + } else { + return ignoreFilePatterns + } +} + // outputDagnode sends dagnode info over the output channel func outputDagnode(out chan interface{}, name string, dn *dag.Node) error { o, err := getOutput(dn) @@ -340,6 +384,14 @@ func (e *hiddenFileError) Error() string { return fmt.Sprintf("%s is a hidden file", e.fileName) } +type ignoreFileError struct { + fileName string +} + +func (e *ignoreFileError) Error() string { + return fmt.Sprintf("%s is an ignored file", e.fileName) +} + type progressReader struct { file files.File out chan interface{} From 500e75669e780a3f1b7ed3e4d80437e9f280ceb5 Mon Sep 17 00:00:00 2001 From: gatesvp Date: Wed, 6 May 2015 14:51:59 -0700 Subject: [PATCH 15/22] check parent paths for .ipfsignore files fix ipfs add to ensure that we are checking parent paths for appropriate .ipfsignore files [ reworded to conform to commit msg guidelines ] --- core/commands/add.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 46ce7dbcab2..a1d7233b21b 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -122,7 +122,15 @@ remains to be implemented. return } - rootnd, err := addFile(n, file, outChan, progress, wrap, hidden, ignoreFilePatterns) + // If the file is not a folder, then let's get the root of that + // folder and attempt to load the appropriate .ipfsignore. + localIgnorePatterns := ignoreFilePatterns + if !file.IsDirectory() { + parentPath := path.Dir(file.FileName()) + localIgnorePatterns = checkForLocalIgnorePatterns(parentPath, ignoreFilePatterns) + } + + rootnd, err := addFile(n, file, outChan, progress, wrap, hidden, localIgnorePatterns) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -299,7 +307,7 @@ func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress boo log.Infof("adding directory: %s", dir.FileName()) // Check for an .ipfsignore file that is local to this Dir and append to the incoming - localIgnorePatterns := checkForLocalIgnorePatterns(dir, ignoreFilePatterns) + localIgnorePatterns := checkForLocalIgnorePatterns(dir.FileName(), ignoreFilePatterns) for { file, err := dir.NextFile() @@ -344,17 +352,17 @@ func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress boo return tree, nil } -func checkForLocalIgnorePatterns(dir files.File, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore { +func checkForLocalIgnorePatterns(dir string, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore { var ignorePathname string - if dir.FileName() == "." { + if dir == "." { ignorePathname = ".ipfsignore" } else { - ignorePathname = path.Join(dir.FileName(), ".ipfsignore") + ignorePathname = path.Join(dir, ".ipfsignore") } localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname) if ignoreErr == nil && localIgnore != nil { - log.Debugf("found ignore file: %s", dir.FileName()) + log.Debugf("found ignore file: %s", dir) return append(ignoreFilePatterns, *localIgnore) } else { return ignoreFilePatterns From 5cb34ee55c668f6842eaaceb38504d60cdaa6baa Mon Sep 17 00:00:00 2001 From: gatesvp Date: Fri, 8 May 2015 13:17:45 -0700 Subject: [PATCH 16/22] Clean-up based on PR #1204 feedback https://github.com/ipfs/go-ipfs/pull/1204 [ reworded to conform to commit msg guidelines ] --- core/commands/add.go | 73 +++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index a1d7233b21b..382163ae6c5 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -3,7 +3,6 @@ package commands import ( "fmt" "io" - "os" "path" "strings" @@ -98,8 +97,8 @@ remains to be implemented. var ignoreFilePatterns []ignore.GitIgnore - // Check the $IPFS_PATH - if ipfs_path := os.Getenv("$IPFS_PATH"); len(ipfs_path) > 0 { + // Check the IPFS_PATH + if ipfs_path := req.Context().ConfigRoot; len(ipfs_path) > 0 { baseFilePattern, err := ignore.CompileIgnoreFile(path.Join(ipfs_path, ".ipfsignore")) if err == nil && baseFilePattern != nil { ignoreFilePatterns = append(ignoreFilePatterns, *baseFilePattern) @@ -130,7 +129,7 @@ remains to be implemented. localIgnorePatterns = checkForLocalIgnorePatterns(parentPath, ignoreFilePatterns) } - rootnd, err := addFile(n, file, outChan, progress, wrap, hidden, localIgnorePatterns) + rootnd, err := addFile(n, addParams{file, outChan, progress, wrap, hidden, localIgnorePatterns}) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -236,6 +235,15 @@ remains to be implemented. Type: AddedObject{}, } +type addParams struct { + file files.File + out chan interface{} + progress bool + wrap bool + hidden bool + ignoreFilePatterns []ignore.GitIgnore +} + func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { node, err := importer.BuildDagFromReader(reader, n.DAG, nil, chunk.DefaultSplitter) if err != nil { @@ -250,41 +258,41 @@ func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { return node, nil } -func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool, wrap bool, hidden bool, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { +func addFile(n *core.IpfsNode, params addParams) (*dag.Node, error) { // Check if file is hidden - if fileIsHidden := files.IsHidden(file); fileIsHidden && !hidden { - log.Debugf("%s is hidden, skipping", file.FileName()) - return nil, &hiddenFileError{file.FileName()} + if fileIsHidden := files.IsHidden(params.file); fileIsHidden && !params.hidden { + log.Debugf("%s is hidden, skipping", params.file.FileName()) + return nil, &hiddenFileError{params.file.FileName()} } // Check for ignore files matches - for i := range ignoreFilePatterns { - if ignoreFilePatterns[i].MatchesPath(file.FileName()) { - log.Debugf("%s is ignored file, skipping", file.FileName()) - return nil, &ignoreFileError{file.FileName()} + for i := range params.ignoreFilePatterns { + if params.ignoreFilePatterns[i].MatchesPath(params.file.FileName()) { + log.Debugf("%s is ignored file, skipping", params.file.FileName()) + return nil, &ignoreFileError{params.file.FileName()} } } // Check if "file" is actually a directory - if file.IsDirectory() { - return addDir(n, file, out, progress, hidden, ignoreFilePatterns) + if params.file.IsDirectory() { + return addDir(n, params) } // if the progress flag was specified, wrap the file so that we can send // progress updates to the client (over the output channel) - var reader io.Reader = file - if progress { - reader = &progressReader{file: file, out: out} + var reader io.Reader = params.file + if params.progress { + reader = &progressReader{file: params.file, out: params.out} } - if wrap { - p, dagnode, err := coreunix.AddWrapped(n, reader, path.Base(file.FileName())) + if params.wrap { + p, dagnode, err := coreunix.AddWrapped(n, reader, path.Base(params.file.FileName())) if err != nil { return nil, err } - out <- &AddedObject{ + params.out <- &AddedObject{ Hash: p, - Name: file.FileName(), + Name: params.file.FileName(), } return dagnode, nil } @@ -294,23 +302,23 @@ func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress b return nil, err } - log.Infof("adding file: %s", file.FileName()) - if err := outputDagnode(out, file.FileName(), dagnode); err != nil { + log.Infof("adding file: %s", params.file.FileName()) + if err := outputDagnode(params.out, params.file.FileName(), dagnode); err != nil { return nil, err } return dagnode, nil } -func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool, hidden bool, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { +func addDir(n *core.IpfsNode, params addParams) (*dag.Node, error) { tree := &dag.Node{Data: ft.FolderPBData()} - log.Infof("adding directory: %s", dir.FileName()) + log.Infof("adding directory: %s", params.file.FileName()) // Check for an .ipfsignore file that is local to this Dir and append to the incoming - localIgnorePatterns := checkForLocalIgnorePatterns(dir.FileName(), ignoreFilePatterns) + localIgnorePatterns := checkForLocalIgnorePatterns(params.file.FileName(), params.ignoreFilePatterns) for { - file, err := dir.NextFile() + file, err := params.file.NextFile() if err != nil && err != io.EOF { return nil, err } @@ -318,7 +326,7 @@ func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress boo break } - node, err := addFile(n, file, out, progress, false, hidden, localIgnorePatterns) + node, err := addFile(n, addParams{file, params.out, params.progress, false, params.hidden, localIgnorePatterns}) if _, ok := err.(*hiddenFileError); ok { // hidden file error, set the node to nil for below node = nil @@ -339,7 +347,7 @@ func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress boo } } - err := outputDagnode(out, dir.FileName(), tree) + err := outputDagnode(params.out, params.file.FileName(), tree) if err != nil { return nil, err } @@ -353,12 +361,7 @@ func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress boo } func checkForLocalIgnorePatterns(dir string, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore { - var ignorePathname string - if dir == "." { - ignorePathname = ".ipfsignore" - } else { - ignorePathname = path.Join(dir, ".ipfsignore") - } + ignorePathname := path.Join(dir, ".ipfsignore") localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname) if ignoreErr == nil && localIgnore != nil { From 441740a3b15d5ef0e626368fa2bba4f44a86ee0e Mon Sep 17 00:00:00 2001 From: gatesvp Date: Tue, 12 May 2015 20:49:53 -0700 Subject: [PATCH 17/22] Clean-up add CLI. Basic params are a struct and addFile/addDir are now methods on the struct [ reworded to conform to commit msg guidelines ] --- core/commands/add.go | 64 ++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 382163ae6c5..46c06c76f42 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -129,7 +129,8 @@ remains to be implemented. localIgnorePatterns = checkForLocalIgnorePatterns(parentPath, ignoreFilePatterns) } - rootnd, err := addFile(n, addParams{file, outChan, progress, wrap, hidden, localIgnorePatterns}) + addParams := adder{n, outChan, progress, wrap, hidden} + rootnd, err := addParams.addFile(file, localIgnorePatterns) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -235,13 +236,12 @@ remains to be implemented. Type: AddedObject{}, } -type addParams struct { - file files.File - out chan interface{} - progress bool - wrap bool - hidden bool - ignoreFilePatterns []ignore.GitIgnore +type adder struct { + node *core.IpfsNode + out chan interface{} + progress bool + wrap bool + hidden bool } func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { @@ -258,67 +258,67 @@ func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { return node, nil } -func addFile(n *core.IpfsNode, params addParams) (*dag.Node, error) { +func (params *adder) addFile(file files.File, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { // Check if file is hidden - if fileIsHidden := files.IsHidden(params.file); fileIsHidden && !params.hidden { - log.Debugf("%s is hidden, skipping", params.file.FileName()) - return nil, &hiddenFileError{params.file.FileName()} + if fileIsHidden := files.IsHidden(file); fileIsHidden && !params.hidden { + log.Debugf("%s is hidden, skipping", file.FileName()) + return nil, &hiddenFileError{file.FileName()} } // Check for ignore files matches - for i := range params.ignoreFilePatterns { - if params.ignoreFilePatterns[i].MatchesPath(params.file.FileName()) { - log.Debugf("%s is ignored file, skipping", params.file.FileName()) - return nil, &ignoreFileError{params.file.FileName()} + for i := range ignoreFilePatterns { + if ignoreFilePatterns[i].MatchesPath(file.FileName()) { + log.Debugf("%s is ignored file, skipping", file.FileName()) + return nil, &ignoreFileError{file.FileName()} } } // Check if "file" is actually a directory - if params.file.IsDirectory() { - return addDir(n, params) + if file.IsDirectory() { + return params.addDir(file, ignoreFilePatterns) } // if the progress flag was specified, wrap the file so that we can send // progress updates to the client (over the output channel) - var reader io.Reader = params.file + var reader io.Reader = file if params.progress { - reader = &progressReader{file: params.file, out: params.out} + reader = &progressReader{file: file, out: params.out} } if params.wrap { - p, dagnode, err := coreunix.AddWrapped(n, reader, path.Base(params.file.FileName())) + p, dagnode, err := coreunix.AddWrapped(params.node, reader, path.Base(file.FileName())) if err != nil { return nil, err } params.out <- &AddedObject{ Hash: p, - Name: params.file.FileName(), + Name: file.FileName(), } return dagnode, nil } - dagnode, err := add(n, reader) + dagnode, err := add(params.node, reader) if err != nil { return nil, err } - log.Infof("adding file: %s", params.file.FileName()) - if err := outputDagnode(params.out, params.file.FileName(), dagnode); err != nil { + log.Infof("adding file: %s", file.FileName()) + if err := outputDagnode(params.out, file.FileName(), dagnode); err != nil { return nil, err } return dagnode, nil } -func addDir(n *core.IpfsNode, params addParams) (*dag.Node, error) { +func (params *adder) addDir(file files.File, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { tree := &dag.Node{Data: ft.FolderPBData()} - log.Infof("adding directory: %s", params.file.FileName()) + log.Infof("adding directory: %s", file.FileName()) // Check for an .ipfsignore file that is local to this Dir and append to the incoming - localIgnorePatterns := checkForLocalIgnorePatterns(params.file.FileName(), params.ignoreFilePatterns) + localIgnorePatterns := checkForLocalIgnorePatterns(file.FileName(), ignoreFilePatterns) for { - file, err := params.file.NextFile() + file, err := file.NextFile() if err != nil && err != io.EOF { return nil, err } @@ -326,7 +326,7 @@ func addDir(n *core.IpfsNode, params addParams) (*dag.Node, error) { break } - node, err := addFile(n, addParams{file, params.out, params.progress, false, params.hidden, localIgnorePatterns}) + node, err := params.addFile(file, localIgnorePatterns) if _, ok := err.(*hiddenFileError); ok { // hidden file error, set the node to nil for below node = nil @@ -347,12 +347,12 @@ func addDir(n *core.IpfsNode, params addParams) (*dag.Node, error) { } } - err := outputDagnode(params.out, params.file.FileName(), tree) + err := outputDagnode(params.out, file.FileName(), tree) if err != nil { return nil, err } - _, err = n.DAG.Add(tree) + _, err = params.node.DAG.Add(tree) if err != nil { return nil, err } From e8d8642f8503234ee6d4d5a9145fc57ac7642c4e Mon Sep 17 00:00:00 2001 From: gatesvp Date: Wed, 13 May 2015 22:19:31 -0700 Subject: [PATCH 18/22] Add recursive check for parent .ipfsignore files --- core/commands/add.go | 45 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 46c06c76f42..170c3cc06d4 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "path" + "path/filepath" "strings" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb" @@ -123,11 +124,7 @@ remains to be implemented. // If the file is not a folder, then let's get the root of that // folder and attempt to load the appropriate .ipfsignore. - localIgnorePatterns := ignoreFilePatterns - if !file.IsDirectory() { - parentPath := path.Dir(file.FileName()) - localIgnorePatterns = checkForLocalIgnorePatterns(parentPath, ignoreFilePatterns) - } + localIgnorePatterns := checkForParentIgnorePatterns(file.FileName(), ignoreFilePatterns) addParams := adder{n, outChan, progress, wrap, hidden} rootnd, err := addParams.addFile(file, localIgnorePatterns) @@ -236,6 +233,7 @@ remains to be implemented. Type: AddedObject{}, } +// Internal structure for holding the switches passed to the `add` call type adder struct { node *core.IpfsNode out chan interface{} @@ -244,6 +242,7 @@ type adder struct { hidden bool } +// Perform the actual add & pin locally, outputting results to reader func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { node, err := importer.BuildDagFromReader(reader, n.DAG, nil, chunk.DefaultSplitter) if err != nil { @@ -258,6 +257,9 @@ func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) { return node, nil } +// Add the given file while respecting the params and ignoreFilePatterns. +// Note that ignoreFilePatterns is not part of the struct as it may change while +// we dig through folders. func (params *adder) addFile(file files.File, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { // Check if file is hidden if fileIsHidden := files.IsHidden(file); fileIsHidden && !params.hidden { @@ -360,7 +362,10 @@ func (params *adder) addDir(file files.File, ignoreFilePatterns []ignore.GitIgno return tree, nil } +// this helper checks the local path for any .ipfsignore file that need to be +// respected. returns the updated or the original GitIgnore. func checkForLocalIgnorePatterns(dir string, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore { + ignorePathname := path.Join(dir, ".ipfsignore") localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname) @@ -372,6 +377,36 @@ func checkForLocalIgnorePatterns(dir string, ignoreFilePatterns []ignore.GitIgno } } +// this helper just walks the parent directories of the given path looking for +// any .ipfsignore files in those directories. +func checkForParentIgnorePatterns(givenPath string, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore { + absolutePath, err := filepath.Abs(givenPath) + + if err != nil { + return ignoreFilePatterns + } + + // break out the absolute path + dir := filepath.Dir(absolutePath) + pathComponents := strings.Split(dir, "\\") + + // We loop through each parent component attempting to find an .ipfsignore file + for index, _ := range pathComponents { + + pathParts := make([]string, len(pathComponents)+1) + copy(pathParts, pathComponents[0:index+1]) + ignorePathname := path.Join(append(pathParts, ".ipfsignore")...) + + localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname) + if ignoreErr == nil && localIgnore != nil { + log.Debugf("found ignore file: %s", dir) + ignoreFilePatterns = append(ignoreFilePatterns, *localIgnore) + } + } + + return ignoreFilePatterns +} + // outputDagnode sends dagnode info over the output channel func outputDagnode(out chan interface{}, name string, dn *dag.Node) error { o, err := getOutput(dn) From ff0d236980e7d7cbb8cb3b4f2779ba439be17ff5 Mon Sep 17 00:00:00 2001 From: gatesvp Date: Wed, 13 May 2015 23:02:37 -0700 Subject: [PATCH 19/22] Respect file path separator on all OSes --- core/commands/add.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/commands/add.go b/core/commands/add.go index 170c3cc06d4..151cf979caa 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -5,6 +5,7 @@ import ( "io" "path" "path/filepath" + "strconv" "strings" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb" @@ -388,7 +389,7 @@ func checkForParentIgnorePatterns(givenPath string, ignoreFilePatterns []ignore. // break out the absolute path dir := filepath.Dir(absolutePath) - pathComponents := strings.Split(dir, "\\") + pathComponents := strings.Split(dir, strconv.QuoteRune(filepath.Separator)) // We loop through each parent component attempting to find an .ipfsignore file for index, _ := range pathComponents { From 954c0db503537dc56f67d1d0f256a7b65a8ec630 Mon Sep 17 00:00:00 2001 From: gatesvp Date: Thu, 14 May 2015 00:52:18 -0700 Subject: [PATCH 20/22] Fix OS-specific slashing without strconv --- core/commands/add.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 151cf979caa..72cd4e619c2 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -5,7 +5,6 @@ import ( "io" "path" "path/filepath" - "strconv" "strings" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb" @@ -389,7 +388,7 @@ func checkForParentIgnorePatterns(givenPath string, ignoreFilePatterns []ignore. // break out the absolute path dir := filepath.Dir(absolutePath) - pathComponents := strings.Split(dir, strconv.QuoteRune(filepath.Separator)) + pathComponents := strings.Split(dir, string(filepath.Separator)) // We loop through each parent component attempting to find an .ipfsignore file for index, _ := range pathComponents { @@ -400,7 +399,7 @@ func checkForParentIgnorePatterns(givenPath string, ignoreFilePatterns []ignore. localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname) if ignoreErr == nil && localIgnore != nil { - log.Debugf("found ignore file: %s", dir) + log.Debugf("found ignore file: %s", ignorePathname) ignoreFilePatterns = append(ignoreFilePatterns, *localIgnore) } } From 53fc454db37308043306833123870114f5255610 Mon Sep 17 00:00:00 2001 From: gatesvp Date: Thu, 14 May 2015 02:54:28 -0400 Subject: [PATCH 21/22] Fix is_hidden on non-Windows --- commands/files/is_hidden.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/files/is_hidden.go b/commands/files/is_hidden.go index 0aeb6ba516e..b0360685b0d 100644 --- a/commands/files/is_hidden.go +++ b/commands/files/is_hidden.go @@ -12,8 +12,8 @@ func IsHidden(f File) bool { fName := filepath.Base(f.FileName()) if strings.HasPrefix(fName, ".") && len(fName) > 1 { - return true, nil + return true } - return false, nil + return false } From 372862bbd13986d2beca5ff9b4a064f25d770bdf Mon Sep 17 00:00:00 2001 From: gatesvp Date: Thu, 14 May 2015 03:16:57 -0400 Subject: [PATCH 22/22] Vendoring for .ipfsignore support --- Godeps/Godeps.json | 4 + .../sabhiram/go-git-ignore/.gitignore | 28 ++ .../sabhiram/go-git-ignore/.travis.yml | 18 ++ .../github.com/sabhiram/go-git-ignore/LICENSE | 22 ++ .../sabhiram/go-git-ignore/README.md | 17 ++ .../sabhiram/go-git-ignore/ignore.go | 172 ++++++++++++ .../sabhiram/go-git-ignore/ignore_test.go | 259 ++++++++++++++++++ core/commands/add.go | 3 +- 8 files changed, 521 insertions(+), 2 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.gitignore create mode 100644 Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.travis.yml create mode 100644 Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/LICENSE create mode 100644 Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/README.md create mode 100644 Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore.go create mode 100644 Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 1ab026f0257..a338c2597e8 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -237,6 +237,10 @@ "ImportPath": "github.com/rs/cors", "Rev": "5e4ce6bc0ecd3472f6f943666d84876691be2ced" }, + { + "ImportPath": "github.com/sabhiram/go-git-ignore", + "Rev": "f9a1328f5fc50414f8751f587774ccd3f49b492b" + }, { "ImportPath": "github.com/steakknife/hamming", "Comment": "0.0.10", diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.gitignore b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.gitignore new file mode 100644 index 00000000000..0e919aff1c2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.gitignore @@ -0,0 +1,28 @@ +# Package test fixtures +test_fixtures + +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.travis.yml b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.travis.yml new file mode 100644 index 00000000000..24ddadf1bf6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.travis.yml @@ -0,0 +1,18 @@ +language: go + +go: + - 1.3 + - tip + +env: + - "PATH=$HOME/gopath/bin:$PATH" + +before_install: + - go get github.com/stretchr/testify/assert + - go get github.com/axw/gocov/gocov + - go get github.com/mattn/goveralls + - if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi + +script: + - go test -v -covermode=count -coverprofile=coverage.out + - goveralls -coverprofile=coverage.out -service travis-ci -repotoken $COVERALLS_TOKEN diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/LICENSE b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/LICENSE new file mode 100644 index 00000000000..c606f49e5c0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Shaba Abhiram + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/README.md b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/README.md new file mode 100644 index 00000000000..fbbb3761dc0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/README.md @@ -0,0 +1,17 @@ +# go-git-ignore + +[![Build Status](https://travis-ci.org/sabhiram/go-git-ignore.svg)](https://travis-ci.org/sabhiram/go-git-ignore) [![Coverage Status](https://coveralls.io/repos/sabhiram/go-git-ignore/badge.png?branch=master)](https://coveralls.io/r/sabhiram/go-git-ignore?branch=master) + +A gitignore parser for `Go` + +## Install + +```shell +go get github.com/sabhiram/go-git-ignore +``` + +## Usage + +```shell +TODO +``` diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore.go b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore.go new file mode 100644 index 00000000000..b15f0d8f2e7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore.go @@ -0,0 +1,172 @@ +/* +ignore is a library which returns a new ignorer object which can +test against various paths. This is particularly useful when trying +to filter files based on a .gitignore document + +The rules for parsing the input file are the same as the ones listed +in the Git docs here: http://git-scm.com/docs/gitignore + +The summarized version of the same has been copied here: + + 1. A blank line matches no files, so it can serve as a separator + for readability. + 2. A line starting with # serves as a comment. Put a backslash ("\") + in front of the first hash for patterns that begin with a hash. + 3. Trailing spaces are ignored unless they are quoted with backslash ("\"). + 4. An optional prefix "!" which negates the pattern; any matching file + excluded by a previous pattern will become included again. It is not + possible to re-include a file if a parent directory of that file is + excluded. Git doesn’t list excluded directories for performance reasons, + so any patterns on contained files have no effect, no matter where they + are defined. Put a backslash ("\") in front of the first "!" for + patterns that begin with a literal "!", for example, "\!important!.txt". + 5. If the pattern ends with a slash, it is removed for the purpose of the + following description, but it would only find a match with a directory. + In other words, foo/ will match a directory foo and paths underneath it, + but will not match a regular file or a symbolic link foo (this is + consistent with the way how pathspec works in general in Git). + 6. If the pattern does not contain a slash /, Git treats it as a shell glob + pattern and checks for a match against the pathname relative to the + location of the .gitignore file (relative to the toplevel of the work + tree if not from a .gitignore file). + 7. Otherwise, Git treats the pattern as a shell glob suitable for + consumption by fnmatch(3) with the FNM_PATHNAME flag: wildcards in the + pattern will not match a / in the pathname. For example, + "Documentation/*.html" matches "Documentation/git.html" but not + "Documentation/ppc/ppc.html" or "tools/perf/Documentation/perf.html". + 8. A leading slash matches the beginning of the pathname. For example, + "/*.c" matches "cat-file.c" but not "mozilla-sha1/sha1.c". + 9. Two consecutive asterisks ("**") in patterns matched against full + pathname may have special meaning: + i. A leading "**" followed by a slash means match in all directories. + For example, "** /foo" matches file or directory "foo" anywhere, + the same as pattern "foo". "** /foo/bar" matches file or directory + "bar" anywhere that is directly under directory "foo". + ii. A trailing "/**" matches everything inside. For example, "abc/**" + matches all files inside directory "abc", relative to the location + of the .gitignore file, with infinite depth. + iii. A slash followed by two consecutive asterisks then a slash matches + zero or more directories. For example, "a/** /b" matches "a/b", + "a/x/b", "a/x/y/b" and so on. + iv. Other consecutive asterisks are considered invalid. */ +package ignore + +import ( + "strings" + "regexp" + "io/ioutil" +) + +// An IgnoreParser is an interface which exposes two methods: +// MatchesPath() - Returns true if the path is targeted by the patterns compiled in the GitIgnore structure +type IgnoreParser interface { + IncludesPath(f string) bool + IgnoresPath(f string) bool + MatchesPath(f string) bool +} + +// GitIgnore is a struct which contains a slice of regexp.Regexp +// patterns +type GitIgnore struct { + patterns []*regexp.Regexp // List of regexp patterns which this ignore file applies + negate []bool // List of booleans which determine if the pattern is negated +} + +// This function pretty much attempts to mimic the parsing rules +// listed above at the start of this file +func getPatternFromLine(line string) (*regexp.Regexp, bool) { + // Strip comments [Rule 2] + if regexp.MustCompile(`^#`).MatchString(line) { return nil, false } + + // Trim string [Rule 3] + // TODO: Hanlde [Rule 3], when the " " is escaped with a \ + line = strings.Trim(line, " ") + + // Exit for no-ops and return nil which will prevent us from + // appending a pattern against this line + if line == "" { return nil, false } + + // TODO: Handle [Rule 4] which negates the match for patterns leading with "!" + negatePattern := false + if string(line[0]) == "!" { + negatePattern = true + line = line[1:] + } + + // Handle [Rule 2, 4], when # or ! is escaped with a \ + // Handle [Rule 4] once we tag negatePattern, strip the leading ! char + if regexp.MustCompile(`^(\#|\!)`).MatchString(line) { + line = line[1:] + } + + // Handle [Rule 8], strip leading / and enforce path checking if its present + if regexp.MustCompile(`^/`).MatchString(line) { + line = "^" + line[1:] + } + + // If we encounter a foo/*.blah in a folder, prepend the ^ char + if regexp.MustCompile(`([^\/+])/.*\*\.`).MatchString(line) { + line = "^" + line + } + + // Handle escaping the "." char + line = regexp.MustCompile(`\.`).ReplaceAllString(line, `\.`) + + // Handle "**" usage (and special case when it is followed by a /) + line = regexp.MustCompile(`\*\*(/|)`).ReplaceAllString(line, `(.+|)`) + + // Handle escaping the "*" char + line = regexp.MustCompile(`\*`).ReplaceAllString(line, `([^\/]+)`) + + + // Temporary regex + expr := line + "(|/.+)$" + pattern, _ := regexp.Compile(expr) + + return pattern, negatePattern +} + +// Accepts a variadic set of strings, and returns a GitIgnore object which +// converts and appends the lines in the input to regexp.Regexp patterns +// held within the GitIgnore objects "patterns" field +func CompileIgnoreLines(lines ...string) (*GitIgnore, error) { + g := new(GitIgnore) + for _, line := range lines { + pattern, negatePattern := getPatternFromLine(line) + if pattern != nil { + g.patterns = append(g.patterns, pattern) + g.negate = append(g.negate, negatePattern) + } + } + return g, nil +} + +// Accepts a ignore file as the input, parses the lines out of the file +// and invokes the CompileIgnoreLines method +func CompileIgnoreFile(fpath string) (*GitIgnore, error) { + buffer, error := ioutil.ReadFile(fpath) + if error == nil { + s := strings.Split(string(buffer), "\n") + return CompileIgnoreLines(s...) + } + return nil, error +} + +// MatchesPath is an interface function for the IgnoreParser interface. +// It returns true if the given GitIgnore structure would target a given +// path string "f" +func (g GitIgnore) MatchesPath(f string) bool { + matchesPath := false + for idx, pattern := range g.patterns { + if pattern.MatchString(f) { + // If this is a regular target (not negated with a gitignore exclude "!" etc) + if !g.negate[idx] { + matchesPath = true + // Negated pattern, and matchesPath is already set + } else if matchesPath { + matchesPath = false + } + } + } + return matchesPath +} diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore_test.go b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore_test.go new file mode 100644 index 00000000000..3893f517d65 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore_test.go @@ -0,0 +1,259 @@ +// Implement tests for the `ignore` library +package ignore + +import ( + "os" + + "io/ioutil" + "path/filepath" + + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + TEST_DIR = "test_fixtures" +) + +// Helper function to setup a test fixture dir and write to +// it a file with the name "fname" and content "content" +func writeFileToTestDir(fname, content string) { + testDirPath := "." + string(filepath.Separator) + TEST_DIR + testFilePath := testDirPath + string(filepath.Separator) + fname + + _ = os.MkdirAll(testDirPath, 0755) + _ = ioutil.WriteFile(testFilePath, []byte(content), os.ModePerm) +} + +func cleanupTestDir() { + _ = os.RemoveAll(fmt.Sprintf(".%s%s", string(filepath.Separator), TEST_DIR)) +} + +// Validate "CompileIgnoreLines()" +func TestCompileIgnoreLines(test *testing.T) { + lines := []string{"abc/def", "a/b/c", "b"} + object, error := CompileIgnoreLines(lines...) + assert.Nil(test, error, "error from CompileIgnoreLines should be nil") + + // MatchesPath + // Paths which are targeted by the above "lines" + assert.Equal(test, true, object.MatchesPath("abc/def/child"), "abc/def/child should match") + assert.Equal(test, true, object.MatchesPath("a/b/c/d"), "a/b/c/d should match") + + // Paths which are not targeted by the above "lines" + assert.Equal(test, false, object.MatchesPath("abc"), "abc should not match") + assert.Equal(test, false, object.MatchesPath("def"), "def should not match") + assert.Equal(test, false, object.MatchesPath("bd"), "bd should not match") + + object, error = CompileIgnoreLines("abc/def", "a/b/c", "b") + assert.Nil(test, error, "error from CompileIgnoreLines should be nil") + + // Paths which are targeted by the above "lines" + assert.Equal(test, true, object.MatchesPath("abc/def/child"), "abc/def/child should match") + assert.Equal(test, true, object.MatchesPath("a/b/c/d"), "a/b/c/d should match") + + // Paths which are not targeted by the above "lines" + assert.Equal(test, false, object.MatchesPath("abc"), "abc should not match") + assert.Equal(test, false, object.MatchesPath("def"), "def should not match") + assert.Equal(test, false, object.MatchesPath("bd"), "bd should not match") +} + +// Validate the invalid files +func TestCompileIgnoreFile_InvalidFile(test *testing.T) { + object, error := CompileIgnoreFile("./test_fixtures/invalid.file") + assert.Nil(test, object, "object should be nil") + assert.NotNil(test, error, "error should be unknown file / dir") +} + +// Validate the an empty files +func TestCompileIgnoreLines_EmptyFile(test *testing.T) { + writeFileToTestDir("test.gitignore", ``) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, false, object.MatchesPath("a"), "should not match any path") + assert.Equal(test, false, object.MatchesPath("a/b"), "should not match any path") + assert.Equal(test, false, object.MatchesPath(".foobar"), "should not match any path") +} + +// Validate the correct handling of the negation operator "!" +func TestCompileIgnoreLines_HandleIncludePattern(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +# exclude everything except directory foo/bar +/* +!/foo +/foo/* +!/foo/bar +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("a"), "a should match") + assert.Equal(test, true, object.MatchesPath("foo/baz"), "foo/baz should match") + assert.Equal(test, false, object.MatchesPath("foo"), "foo should not match") + assert.Equal(test, false, object.MatchesPath("/foo/bar"), "/foo/bar should not match") +} + +// Validate the correct handling of comments and empty lines +func TestCompileIgnoreLines_HandleSpaces(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +# +# A comment + +# Another comment + + + # Invalid Comment + +abc/def +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, 2, len(object.patterns), "should have two regex pattern") + assert.Equal(test, false, object.MatchesPath("abc/abc"), "/abc/abc should not match") + assert.Equal(test, true, object.MatchesPath("abc/def"), "/abc/def should match") +} + +// Validate the correct handling of leading / chars +func TestCompileIgnoreLines_HandleLeadingSlash(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +/a/b/c +d/e/f +/g +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, 3, len(object.patterns), "should have 3 regex patterns") + assert.Equal(test, true, object.MatchesPath("a/b/c"), "a/b/c should match") + assert.Equal(test, true, object.MatchesPath("a/b/c/d"), "a/b/c/d should match") + assert.Equal(test, true, object.MatchesPath("d/e/f"), "d/e/f should match") + assert.Equal(test, true, object.MatchesPath("g"), "g should match") +} + +// Validate the correct handling of files starting with # or ! +func TestCompileIgnoreLines_HandleLeadingSpecialChars(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +# Comment +\#file.txt +\!file.txt +file.txt +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("#file.txt"), "#file.txt should match") + assert.Equal(test, true, object.MatchesPath("!file.txt"), "!file.txt should match") + assert.Equal(test, true, object.MatchesPath("a/!file.txt"), "a/!file.txt should match") + assert.Equal(test, true, object.MatchesPath("file.txt"), "file.txt should match") + assert.Equal(test, true, object.MatchesPath("a/file.txt"), "a/file.txt should match") + assert.Equal(test, false, object.MatchesPath("file2.txt"), "file2.txt should not match") + +} + +// Validate the correct handling matching files only within a given folder +func TestCompileIgnoreLines_HandleAllFilesInDir(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +Documentation/*.html +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("Documentation/git.html"), "Documentation/git.html should match") + assert.Equal(test, false, object.MatchesPath("Documentation/ppc/ppc.html"), "Documentation/ppc/ppc.html should not match") + assert.Equal(test, false, object.MatchesPath("tools/perf/Documentation/perf.html"), "tools/perf/Documentation/perf.html should not match") +} + +// Validate the correct handling of "**" +func TestCompileIgnoreLines_HandleDoubleStar(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +**/foo +bar +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("foo"), "foo should match") + assert.Equal(test, true, object.MatchesPath("baz/foo"), "baz/foo should match") + assert.Equal(test, true, object.MatchesPath("bar"), "bar should match") + assert.Equal(test, true, object.MatchesPath("baz/bar"), "baz/bar should match") +} + +// Validate the correct handling of leading slash +func TestCompileIgnoreLines_HandleLeadingSlashPath(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +/*.c +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("hello.c"), "hello.c should match") + assert.Equal(test, false, object.MatchesPath("foo/hello.c"), "foo/hello.c should not match") +} + +func ExampleCompileIgnoreLines() { + ignoreObject, error := CompileIgnoreLines([]string{"node_modules", "*.out", "foo/*.c"}...) + if error != nil { + panic("Error when compiling ignore lines: " + error.Error()) + } + + // You can test the ignoreObject against various paths using the + // "MatchesPath()" interface method. This pretty much is up to + // the users interpretation. In the case of a ".gitignore" file, + // a "match" would indicate that a given path would be ignored. + fmt.Println(ignoreObject.MatchesPath("node_modules/test/foo.js")) + fmt.Println(ignoreObject.MatchesPath("node_modules2/test.out")) + fmt.Println(ignoreObject.MatchesPath("test/foo.js")) + + // Output: + // true + // true + // false +} + +func TestCompileIgnoreLines_CheckNestedDotFiles(test *testing.T) { + lines := []string{ + "**/external/**/*.md", + "**/external/**/*.json", + "**/external/**/*.gzip", + "**/external/**/.*ignore", + + "**/external/foobar/*.css", + "**/external/barfoo/less", + "**/external/barfoo/scss", + } + object, error := CompileIgnoreLines(lines...) + assert.Nil(test, error, "error from CompileIgnoreLines should be nil") + assert.NotNil(test, object, "returned object should not be nil") + + assert.Equal(test, true, object.MatchesPath("external/foobar/angular.foo.css"), "external/foobar/angular.foo.css") + assert.Equal(test, true, object.MatchesPath("external/barfoo/.gitignore"), "external/barfoo/.gitignore") + assert.Equal(test, true, object.MatchesPath("external/barfoo/.bower.json"), "external/barfoo/.bower.json") +} diff --git a/core/commands/add.go b/core/commands/add.go index 72cd4e619c2..c1dc15bb1bd 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -8,9 +8,8 @@ import ( "strings" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb" + ignore "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" - ignore "github.com/sabhiram/go-git-ignore" - cmds "github.com/ipfs/go-ipfs/commands" files "github.com/ipfs/go-ipfs/commands/files" core "github.com/ipfs/go-ipfs/core"