diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 32172888809..692d9b76433 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -256,6 +256,10 @@ "Rev": "5e4ce6bc0ecd3472f6f943666d84876691be2ced" }, { + "ImportPath": "github.com/sabhiram/go-git-ignore", + "Rev": "f9a1328f5fc50414f8751f587774ccd3f49b492b" + }, + { "ImportPath": "github.com/satori/go.uuid", "Rev": "7c7f2020c4c9491594b85767967f4619c2fa75f9" }, diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.gitignore b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.gitignore new file mode 100644 index 00000000000..0e919aff1c2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.gitignore @@ -0,0 +1,28 @@ +# Package test fixtures +test_fixtures + +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.travis.yml b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.travis.yml new file mode 100644 index 00000000000..24ddadf1bf6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/.travis.yml @@ -0,0 +1,18 @@ +language: go + +go: + - 1.3 + - tip + +env: + - "PATH=$HOME/gopath/bin:$PATH" + +before_install: + - go get github.com/stretchr/testify/assert + - go get github.com/axw/gocov/gocov + - go get github.com/mattn/goveralls + - if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi + +script: + - go test -v -covermode=count -coverprofile=coverage.out + - goveralls -coverprofile=coverage.out -service travis-ci -repotoken $COVERALLS_TOKEN diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/LICENSE b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/LICENSE new file mode 100644 index 00000000000..c606f49e5c0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Shaba Abhiram + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/README.md b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/README.md new file mode 100644 index 00000000000..fbbb3761dc0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/README.md @@ -0,0 +1,17 @@ +# go-git-ignore + +[![Build Status](https://travis-ci.org/sabhiram/go-git-ignore.svg)](https://travis-ci.org/sabhiram/go-git-ignore) [![Coverage Status](https://coveralls.io/repos/sabhiram/go-git-ignore/badge.png?branch=master)](https://coveralls.io/r/sabhiram/go-git-ignore?branch=master) + +A gitignore parser for `Go` + +## Install + +```shell +go get github.com/sabhiram/go-git-ignore +``` + +## Usage + +```shell +TODO +``` diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore.go b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore.go new file mode 100644 index 00000000000..b15f0d8f2e7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore.go @@ -0,0 +1,172 @@ +/* +ignore is a library which returns a new ignorer object which can +test against various paths. This is particularly useful when trying +to filter files based on a .gitignore document + +The rules for parsing the input file are the same as the ones listed +in the Git docs here: http://git-scm.com/docs/gitignore + +The summarized version of the same has been copied here: + + 1. A blank line matches no files, so it can serve as a separator + for readability. + 2. A line starting with # serves as a comment. Put a backslash ("\") + in front of the first hash for patterns that begin with a hash. + 3. Trailing spaces are ignored unless they are quoted with backslash ("\"). + 4. An optional prefix "!" which negates the pattern; any matching file + excluded by a previous pattern will become included again. It is not + possible to re-include a file if a parent directory of that file is + excluded. Git doesn’t list excluded directories for performance reasons, + so any patterns on contained files have no effect, no matter where they + are defined. Put a backslash ("\") in front of the first "!" for + patterns that begin with a literal "!", for example, "\!important!.txt". + 5. If the pattern ends with a slash, it is removed for the purpose of the + following description, but it would only find a match with a directory. + In other words, foo/ will match a directory foo and paths underneath it, + but will not match a regular file or a symbolic link foo (this is + consistent with the way how pathspec works in general in Git). + 6. If the pattern does not contain a slash /, Git treats it as a shell glob + pattern and checks for a match against the pathname relative to the + location of the .gitignore file (relative to the toplevel of the work + tree if not from a .gitignore file). + 7. Otherwise, Git treats the pattern as a shell glob suitable for + consumption by fnmatch(3) with the FNM_PATHNAME flag: wildcards in the + pattern will not match a / in the pathname. For example, + "Documentation/*.html" matches "Documentation/git.html" but not + "Documentation/ppc/ppc.html" or "tools/perf/Documentation/perf.html". + 8. A leading slash matches the beginning of the pathname. For example, + "/*.c" matches "cat-file.c" but not "mozilla-sha1/sha1.c". + 9. Two consecutive asterisks ("**") in patterns matched against full + pathname may have special meaning: + i. A leading "**" followed by a slash means match in all directories. + For example, "** /foo" matches file or directory "foo" anywhere, + the same as pattern "foo". "** /foo/bar" matches file or directory + "bar" anywhere that is directly under directory "foo". + ii. A trailing "/**" matches everything inside. For example, "abc/**" + matches all files inside directory "abc", relative to the location + of the .gitignore file, with infinite depth. + iii. A slash followed by two consecutive asterisks then a slash matches + zero or more directories. For example, "a/** /b" matches "a/b", + "a/x/b", "a/x/y/b" and so on. + iv. Other consecutive asterisks are considered invalid. */ +package ignore + +import ( + "strings" + "regexp" + "io/ioutil" +) + +// An IgnoreParser is an interface which exposes two methods: +// MatchesPath() - Returns true if the path is targeted by the patterns compiled in the GitIgnore structure +type IgnoreParser interface { + IncludesPath(f string) bool + IgnoresPath(f string) bool + MatchesPath(f string) bool +} + +// GitIgnore is a struct which contains a slice of regexp.Regexp +// patterns +type GitIgnore struct { + patterns []*regexp.Regexp // List of regexp patterns which this ignore file applies + negate []bool // List of booleans which determine if the pattern is negated +} + +// This function pretty much attempts to mimic the parsing rules +// listed above at the start of this file +func getPatternFromLine(line string) (*regexp.Regexp, bool) { + // Strip comments [Rule 2] + if regexp.MustCompile(`^#`).MatchString(line) { return nil, false } + + // Trim string [Rule 3] + // TODO: Hanlde [Rule 3], when the " " is escaped with a \ + line = strings.Trim(line, " ") + + // Exit for no-ops and return nil which will prevent us from + // appending a pattern against this line + if line == "" { return nil, false } + + // TODO: Handle [Rule 4] which negates the match for patterns leading with "!" + negatePattern := false + if string(line[0]) == "!" { + negatePattern = true + line = line[1:] + } + + // Handle [Rule 2, 4], when # or ! is escaped with a \ + // Handle [Rule 4] once we tag negatePattern, strip the leading ! char + if regexp.MustCompile(`^(\#|\!)`).MatchString(line) { + line = line[1:] + } + + // Handle [Rule 8], strip leading / and enforce path checking if its present + if regexp.MustCompile(`^/`).MatchString(line) { + line = "^" + line[1:] + } + + // If we encounter a foo/*.blah in a folder, prepend the ^ char + if regexp.MustCompile(`([^\/+])/.*\*\.`).MatchString(line) { + line = "^" + line + } + + // Handle escaping the "." char + line = regexp.MustCompile(`\.`).ReplaceAllString(line, `\.`) + + // Handle "**" usage (and special case when it is followed by a /) + line = regexp.MustCompile(`\*\*(/|)`).ReplaceAllString(line, `(.+|)`) + + // Handle escaping the "*" char + line = regexp.MustCompile(`\*`).ReplaceAllString(line, `([^\/]+)`) + + + // Temporary regex + expr := line + "(|/.+)$" + pattern, _ := regexp.Compile(expr) + + return pattern, negatePattern +} + +// Accepts a variadic set of strings, and returns a GitIgnore object which +// converts and appends the lines in the input to regexp.Regexp patterns +// held within the GitIgnore objects "patterns" field +func CompileIgnoreLines(lines ...string) (*GitIgnore, error) { + g := new(GitIgnore) + for _, line := range lines { + pattern, negatePattern := getPatternFromLine(line) + if pattern != nil { + g.patterns = append(g.patterns, pattern) + g.negate = append(g.negate, negatePattern) + } + } + return g, nil +} + +// Accepts a ignore file as the input, parses the lines out of the file +// and invokes the CompileIgnoreLines method +func CompileIgnoreFile(fpath string) (*GitIgnore, error) { + buffer, error := ioutil.ReadFile(fpath) + if error == nil { + s := strings.Split(string(buffer), "\n") + return CompileIgnoreLines(s...) + } + return nil, error +} + +// MatchesPath is an interface function for the IgnoreParser interface. +// It returns true if the given GitIgnore structure would target a given +// path string "f" +func (g GitIgnore) MatchesPath(f string) bool { + matchesPath := false + for idx, pattern := range g.patterns { + if pattern.MatchString(f) { + // If this is a regular target (not negated with a gitignore exclude "!" etc) + if !g.negate[idx] { + matchesPath = true + // Negated pattern, and matchesPath is already set + } else if matchesPath { + matchesPath = false + } + } + } + return matchesPath +} diff --git a/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore_test.go b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore_test.go new file mode 100644 index 00000000000..3893f517d65 --- /dev/null +++ b/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore/ignore_test.go @@ -0,0 +1,259 @@ +// Implement tests for the `ignore` library +package ignore + +import ( + "os" + + "io/ioutil" + "path/filepath" + + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + TEST_DIR = "test_fixtures" +) + +// Helper function to setup a test fixture dir and write to +// it a file with the name "fname" and content "content" +func writeFileToTestDir(fname, content string) { + testDirPath := "." + string(filepath.Separator) + TEST_DIR + testFilePath := testDirPath + string(filepath.Separator) + fname + + _ = os.MkdirAll(testDirPath, 0755) + _ = ioutil.WriteFile(testFilePath, []byte(content), os.ModePerm) +} + +func cleanupTestDir() { + _ = os.RemoveAll(fmt.Sprintf(".%s%s", string(filepath.Separator), TEST_DIR)) +} + +// Validate "CompileIgnoreLines()" +func TestCompileIgnoreLines(test *testing.T) { + lines := []string{"abc/def", "a/b/c", "b"} + object, error := CompileIgnoreLines(lines...) + assert.Nil(test, error, "error from CompileIgnoreLines should be nil") + + // MatchesPath + // Paths which are targeted by the above "lines" + assert.Equal(test, true, object.MatchesPath("abc/def/child"), "abc/def/child should match") + assert.Equal(test, true, object.MatchesPath("a/b/c/d"), "a/b/c/d should match") + + // Paths which are not targeted by the above "lines" + assert.Equal(test, false, object.MatchesPath("abc"), "abc should not match") + assert.Equal(test, false, object.MatchesPath("def"), "def should not match") + assert.Equal(test, false, object.MatchesPath("bd"), "bd should not match") + + object, error = CompileIgnoreLines("abc/def", "a/b/c", "b") + assert.Nil(test, error, "error from CompileIgnoreLines should be nil") + + // Paths which are targeted by the above "lines" + assert.Equal(test, true, object.MatchesPath("abc/def/child"), "abc/def/child should match") + assert.Equal(test, true, object.MatchesPath("a/b/c/d"), "a/b/c/d should match") + + // Paths which are not targeted by the above "lines" + assert.Equal(test, false, object.MatchesPath("abc"), "abc should not match") + assert.Equal(test, false, object.MatchesPath("def"), "def should not match") + assert.Equal(test, false, object.MatchesPath("bd"), "bd should not match") +} + +// Validate the invalid files +func TestCompileIgnoreFile_InvalidFile(test *testing.T) { + object, error := CompileIgnoreFile("./test_fixtures/invalid.file") + assert.Nil(test, object, "object should be nil") + assert.NotNil(test, error, "error should be unknown file / dir") +} + +// Validate the an empty files +func TestCompileIgnoreLines_EmptyFile(test *testing.T) { + writeFileToTestDir("test.gitignore", ``) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, false, object.MatchesPath("a"), "should not match any path") + assert.Equal(test, false, object.MatchesPath("a/b"), "should not match any path") + assert.Equal(test, false, object.MatchesPath(".foobar"), "should not match any path") +} + +// Validate the correct handling of the negation operator "!" +func TestCompileIgnoreLines_HandleIncludePattern(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +# exclude everything except directory foo/bar +/* +!/foo +/foo/* +!/foo/bar +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("a"), "a should match") + assert.Equal(test, true, object.MatchesPath("foo/baz"), "foo/baz should match") + assert.Equal(test, false, object.MatchesPath("foo"), "foo should not match") + assert.Equal(test, false, object.MatchesPath("/foo/bar"), "/foo/bar should not match") +} + +// Validate the correct handling of comments and empty lines +func TestCompileIgnoreLines_HandleSpaces(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +# +# A comment + +# Another comment + + + # Invalid Comment + +abc/def +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, 2, len(object.patterns), "should have two regex pattern") + assert.Equal(test, false, object.MatchesPath("abc/abc"), "/abc/abc should not match") + assert.Equal(test, true, object.MatchesPath("abc/def"), "/abc/def should match") +} + +// Validate the correct handling of leading / chars +func TestCompileIgnoreLines_HandleLeadingSlash(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +/a/b/c +d/e/f +/g +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, 3, len(object.patterns), "should have 3 regex patterns") + assert.Equal(test, true, object.MatchesPath("a/b/c"), "a/b/c should match") + assert.Equal(test, true, object.MatchesPath("a/b/c/d"), "a/b/c/d should match") + assert.Equal(test, true, object.MatchesPath("d/e/f"), "d/e/f should match") + assert.Equal(test, true, object.MatchesPath("g"), "g should match") +} + +// Validate the correct handling of files starting with # or ! +func TestCompileIgnoreLines_HandleLeadingSpecialChars(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +# Comment +\#file.txt +\!file.txt +file.txt +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("#file.txt"), "#file.txt should match") + assert.Equal(test, true, object.MatchesPath("!file.txt"), "!file.txt should match") + assert.Equal(test, true, object.MatchesPath("a/!file.txt"), "a/!file.txt should match") + assert.Equal(test, true, object.MatchesPath("file.txt"), "file.txt should match") + assert.Equal(test, true, object.MatchesPath("a/file.txt"), "a/file.txt should match") + assert.Equal(test, false, object.MatchesPath("file2.txt"), "file2.txt should not match") + +} + +// Validate the correct handling matching files only within a given folder +func TestCompileIgnoreLines_HandleAllFilesInDir(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +Documentation/*.html +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("Documentation/git.html"), "Documentation/git.html should match") + assert.Equal(test, false, object.MatchesPath("Documentation/ppc/ppc.html"), "Documentation/ppc/ppc.html should not match") + assert.Equal(test, false, object.MatchesPath("tools/perf/Documentation/perf.html"), "tools/perf/Documentation/perf.html should not match") +} + +// Validate the correct handling of "**" +func TestCompileIgnoreLines_HandleDoubleStar(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +**/foo +bar +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("foo"), "foo should match") + assert.Equal(test, true, object.MatchesPath("baz/foo"), "baz/foo should match") + assert.Equal(test, true, object.MatchesPath("bar"), "bar should match") + assert.Equal(test, true, object.MatchesPath("baz/bar"), "baz/bar should match") +} + +// Validate the correct handling of leading slash +func TestCompileIgnoreLines_HandleLeadingSlashPath(test *testing.T) { + writeFileToTestDir("test.gitignore", ` +/*.c +`) + defer cleanupTestDir() + + object, error := CompileIgnoreFile("./test_fixtures/test.gitignore") + assert.Nil(test, error, "error should be nil") + assert.NotNil(test, object, "object should not be nil") + + assert.Equal(test, true, object.MatchesPath("hello.c"), "hello.c should match") + assert.Equal(test, false, object.MatchesPath("foo/hello.c"), "foo/hello.c should not match") +} + +func ExampleCompileIgnoreLines() { + ignoreObject, error := CompileIgnoreLines([]string{"node_modules", "*.out", "foo/*.c"}...) + if error != nil { + panic("Error when compiling ignore lines: " + error.Error()) + } + + // You can test the ignoreObject against various paths using the + // "MatchesPath()" interface method. This pretty much is up to + // the users interpretation. In the case of a ".gitignore" file, + // a "match" would indicate that a given path would be ignored. + fmt.Println(ignoreObject.MatchesPath("node_modules/test/foo.js")) + fmt.Println(ignoreObject.MatchesPath("node_modules2/test.out")) + fmt.Println(ignoreObject.MatchesPath("test/foo.js")) + + // Output: + // true + // true + // false +} + +func TestCompileIgnoreLines_CheckNestedDotFiles(test *testing.T) { + lines := []string{ + "**/external/**/*.md", + "**/external/**/*.json", + "**/external/**/*.gzip", + "**/external/**/.*ignore", + + "**/external/foobar/*.css", + "**/external/barfoo/less", + "**/external/barfoo/scss", + } + object, error := CompileIgnoreLines(lines...) + assert.Nil(test, error, "error from CompileIgnoreLines should be nil") + assert.NotNil(test, object, "returned object should not be nil") + + assert.Equal(test, true, object.MatchesPath("external/foobar/angular.foo.css"), "external/foobar/angular.foo.css") + assert.Equal(test, true, object.MatchesPath("external/barfoo/.gitignore"), "external/barfoo/.gitignore") + assert.Equal(test, true, object.MatchesPath("external/barfoo/.bower.json"), "external/barfoo/.bower.json") +} diff --git a/commands/files/is_hidden.go b/commands/files/is_hidden.go new file mode 100644 index 00000000000..b0360685b0d --- /dev/null +++ b/commands/files/is_hidden.go @@ -0,0 +1,19 @@ +// +build !windows + +package files + +import ( + "path/filepath" + "strings" +) + +func IsHidden(f File) bool { + + fName := filepath.Base(f.FileName()) + + if strings.HasPrefix(fName, ".") && len(fName) > 1 { + return true + } + + return false +} diff --git a/commands/files/is_hidden_windows.go b/commands/files/is_hidden_windows.go new file mode 100644 index 00000000000..5d263931033 --- /dev/null +++ b/commands/files/is_hidden_windows.go @@ -0,0 +1,29 @@ +// +build windows + +package files + +import ( + "path/filepath" + "strings" + "syscall" +) + +func IsHidden(f File) bool { + + fName := filepath.Base(f.FileName()) + + if strings.HasPrefix(fName, ".") && len(fName) > 1 { + return true + } + + p, e := syscall.UTF16PtrFromString(f.FileName()) + if e != nil { + return false + } + + attrs, e := syscall.GetFileAttributes(p) + if e != nil { + return false + } + return attrs&syscall.FILE_ATTRIBUTE_HIDDEN != 0 +} diff --git a/commands/files/serialfile.go b/commands/files/serialfile.go index aeba01fa7ed..461bde33600 100644 --- a/commands/files/serialfile.go +++ b/commands/files/serialfile.go @@ -3,7 +3,7 @@ package files import ( "io" "os" - fp "path" + fp "path/filepath" "sort" "syscall" ) diff --git a/core/commands/add.go b/core/commands/add.go index 721858c6288..695635b05f1 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -4,10 +4,12 @@ import ( "fmt" "io" "path" + "path/filepath" "strings" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb" - + ignore "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/sabhiram/go-git-ignore" + //context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" cmds "github.com/ipfs/go-ipfs/commands" files "github.com/ipfs/go-ipfs/commands/files" core "github.com/ipfs/go-ipfs/core" @@ -30,6 +32,7 @@ const ( progressOptionName = "progress" trickleOptionName = "trickle" wrapOptionName = "wrap-with-directory" + hiddenOptionName = "hidden" ) type AddedObject struct { @@ -57,6 +60,7 @@ remains to be implemented. cmds.BoolOption("quiet", "q", "Write minimal output"), cmds.BoolOption(progressOptionName, "p", "Stream progress data"), cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object"), + cmds.BoolOption(hiddenOptionName, "Include files that are hidden"), cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation"), cmds.BoolOption("only-hash", "n", "Only chunk and hash the specified content, don't write to disk"), }, @@ -94,6 +98,17 @@ remains to be implemented. trickle, _, _ := req.Option(trickleOptionName).Bool() wrap, _, _ := req.Option(wrapOptionName).Bool() hash, _, _ := req.Option("only-hash").Bool() + hidden, _, _ := req.Option(hiddenOptionName).Bool() + + var ignoreFilePatterns []ignore.GitIgnore + + // Check the IPFS_PATH + if ipfs_path := req.Context().ConfigRoot; len(ipfs_path) > 0 { + baseFilePattern, err := ignore.CompileIgnoreFile(path.Join(ipfs_path, ".ipfsignore")) + if err == nil && baseFilePattern != nil { + ignoreFilePatterns = append(ignoreFilePatterns, *baseFilePattern) + } + } if hash { nilnode, err := core.NewNodeBuilder().NilRepo().Build(n.Context()) @@ -120,7 +135,12 @@ remains to be implemented. return } - rootnd, err := addFile(n, file, outChan, progress, wrap, trickle) + // If the file is not a folder, then let's get the root of that + // folder and attempt to load the appropriate .ipfsignore. + localIgnorePatterns := checkForParentIgnorePatterns(file.FileName(), ignoreFilePatterns) + + addParams := adder{n, outChan, progress, wrap, hidden, trickle} + rootnd, err := addParams.addFile(file, localIgnorePatterns) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -230,6 +250,17 @@ remains to be implemented. Type: AddedObject{}, } +// Internal structure for holding the switches passed to the `add` call +type adder struct { + node *core.IpfsNode + out chan interface{} + progress bool + wrap bool + hidden bool + trickle bool +} + +// Perform the actual add & pin locally, outputting results to reader func add(n *core.IpfsNode, reader io.Reader, useTrickle bool) (*dag.Node, error) { var node *dag.Node var err error @@ -256,49 +287,69 @@ func add(n *core.IpfsNode, reader io.Reader, useTrickle bool) (*dag.Node, error) return node, nil } -func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool, wrap bool, useTrickle bool) (*dag.Node, error) { +// Add the given file while respecting the params and ignoreFilePatterns. +// Note that ignoreFilePatterns is not part of the struct as it may change while +// we dig through folders. +func (params *adder) addFile(file files.File, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { + // Check if file is hidden + if fileIsHidden := files.IsHidden(file); fileIsHidden && !params.hidden { + log.Debugf("%s is hidden, skipping", file.FileName()) + return nil, &hiddenFileError{file.FileName()} + } + + // Check for ignore files matches + for i := range ignoreFilePatterns { + if ignoreFilePatterns[i].MatchesPath(file.FileName()) { + log.Debugf("%s is ignored file, skipping", file.FileName()) + return nil, &ignoreFileError{file.FileName()} + } + } + + // Check if "file" is actually a directory if file.IsDirectory() { - return addDir(n, file, out, progress, useTrickle) + return params.addDir(file, ignoreFilePatterns) } // if the progress flag was specified, wrap the file so that we can send // progress updates to the client (over the output channel) var reader io.Reader = file - if progress { - reader = &progressReader{file: file, out: out} + if params.progress { + reader = &progressReader{file: file, out: params.out} } - if wrap { - p, dagnode, err := coreunix.AddWrapped(n, reader, path.Base(file.FileName())) + if params.wrap { + p, dagnode, err := coreunix.AddWrapped(params.node, reader, path.Base(file.FileName())) if err != nil { return nil, err } - out <- &AddedObject{ + params.out <- &AddedObject{ Hash: p, Name: file.FileName(), } return dagnode, nil } - dagnode, err := add(n, reader, useTrickle) + dagnode, err := add(params.node, reader, params.trickle) if err != nil { return nil, err } log.Infof("adding file: %s", file.FileName()) - if err := outputDagnode(out, file.FileName(), dagnode); err != nil { + if err := outputDagnode(params.out, file.FileName(), dagnode); err != nil { return nil, err } return dagnode, nil } -func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool, useTrickle bool) (*dag.Node, error) { - log.Infof("adding directory: %s", dir.FileName()) - +func (params *adder) addDir(file files.File, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) { tree := &dag.Node{Data: ft.FolderPBData()} + log.Infof("adding directory: %s", file.FileName()) + + // Check for an .ipfsignore file that is local to this Dir and append to the incoming + localIgnorePatterns := checkForLocalIgnorePatterns(file.FileName(), ignoreFilePatterns) for { - file, err := dir.NextFile() + file, err := file.NextFile() if err != nil && err != io.EOF { return nil, err } @@ -306,34 +357,88 @@ func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress boo break } - node, err := addFile(n, file, out, progress, false, useTrickle) - if err != nil { + node, err := params.addFile(file, localIgnorePatterns) + if _, ok := err.(*hiddenFileError); ok { + // hidden file error, set the node to nil for below + node = nil + } else if _, ok := err.(*ignoreFileError); ok { + // ignore file error, set the node to nil for below + node = nil + } else if err != nil { return nil, err } - _, name := path.Split(file.FileName()) + if node != nil { + _, name := path.Split(file.FileName()) - err = tree.AddNodeLink(name, node) - if err != nil { - return nil, err + err = tree.AddNodeLink(name, node) + if err != nil { + return nil, err + } } } - err := outputDagnode(out, dir.FileName(), tree) + err := outputDagnode(params.out, file.FileName(), tree) if err != nil { return nil, err } - k, err := n.DAG.Add(tree) + k, err := params.node.DAG.Add(tree) if err != nil { return nil, err } - n.Pinning.GetManual().PinWithMode(k, pin.Indirect) + params.node.Pinning.GetManual().PinWithMode(k, pin.Indirect) return tree, nil } +// this helper checks the local path for any .ipfsignore file that need to be +// respected. returns the updated or the original GitIgnore. +func checkForLocalIgnorePatterns(dir string, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore { + + ignorePathname := path.Join(dir, ".ipfsignore") + + localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname) + if ignoreErr == nil && localIgnore != nil { + absoluteDir, _ := filepath.Abs(ignorePathname) // ignoring error it's just loggin + log.Debugf("found local ignore file: %s", absoluteDir) + return append(ignoreFilePatterns, *localIgnore) + } else { + return ignoreFilePatterns + } +} + +// this helper just walks the parent directories of the given path looking for +// any .ipfsignore files in those directories. +func checkForParentIgnorePatterns(givenPath string, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore { + absolutePath, err := filepath.Abs(givenPath) + + if err != nil { + return ignoreFilePatterns + } + + // break out the absolute path + dir := filepath.Dir(absolutePath) + pathComponents := strings.Split(dir, string(filepath.Separator)) + + // We loop through each parent component attempting to find an .ipfsignore file + for index, _ := range pathComponents { + + pathParts := make([]string, index+1) + copy(pathParts, pathComponents[0:index+1]) + ignorePathname := path.Clean(strings.Join(append(pathParts, ".ipfsignore"), string(filepath.Separator))) + + localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname) + if ignoreErr == nil && localIgnore != nil { + log.Debugf("found parent ignore file: %s", ignorePathname) + ignoreFilePatterns = append(ignoreFilePatterns, *localIgnore) + } + } + + return ignoreFilePatterns +} + // outputDagnode sends dagnode info over the output channel func outputDagnode(out chan interface{}, name string, dn *dag.Node) error { o, err := getOutput(dn) @@ -349,6 +454,22 @@ func outputDagnode(out chan interface{}, name string, dn *dag.Node) error { return nil } +type hiddenFileError struct { + fileName string +} + +func (e *hiddenFileError) Error() string { + return fmt.Sprintf("%s is a hidden file", e.fileName) +} + +type ignoreFileError struct { + fileName string +} + +func (e *ignoreFileError) Error() string { + return fmt.Sprintf("%s is an ignored file", e.fileName) +} + type progressReader struct { file files.File out chan interface{} diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index 0e3e6fb08fd..2b5755b0417 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -140,11 +140,71 @@ test_expect_success "'ipfs add -r' output looks good" ' PLANETS="QmWSgS32xQEcXMeqd3YPJLrNBLSdsfYCep2U7CFkyrjXwY" && MARS="QmPrrHqJzto9m7SyiRzarwkqPcCSsKR2EB1AyqJfe8L8tN" && VENUS="QmU5kp3BH3B8tnWUU2Pikdb2maksBNkb92FHRr56hyghh4" && - echo "added $MARS mountdir/planets/mars.txt" >expected && - echo "added $VENUS mountdir/planets/venus.txt" >>expected && - echo "added $PLANETS mountdir/planets" >>expected && + echo "added $MARS mountdir/planets/mars.txt" > expected && + echo "added $VENUS mountdir/planets/venus.txt" >> expected && + echo "added $PLANETS mountdir/planets" >> expected && test_cmp expected actual ' +test_expect_success "'ipfs add -r' treats . as ignore" ' + echo "Hello Mars!" >mountdir/planets/mars.txt && + echo "Hello Venus!" >mountdir/planets/venus.txt && + echo "ignore*" >mountdir/planets/.ipfsignore && + ipfs add -r mountdir/planets >actual + PLANETS="QmWSgS32xQEcXMeqd3YPJLrNBLSdsfYCep2U7CFkyrjXwY" && + MARS="QmPrrHqJzto9m7SyiRzarwkqPcCSsKR2EB1AyqJfe8L8tN" && + VENUS="QmU5kp3BH3B8tnWUU2Pikdb2maksBNkb92FHRr56hyghh4" && + echo "added $MARS mountdir/planets/mars.txt" > expected && + echo "added $VENUS mountdir/planets/venus.txt" >> expected && + echo "added $PLANETS mountdir/planets" >> expected && + test_cmp expected actual +' + +test_expect_success "'ipfs add -r --hidden' will include hidden file" ' + ipfs add -r --hidden mountdir/planets > actual + IGNORE="QmcPE2m7ori3WozZT6HzAxLP7nSUNAfgRENUEkKpfvAerJ" && + PLANETS="QmQALn8EZGDAPUKijt56mYLdqTNvxMAZBAoQpDjV7G1A8D" && + MARS="QmPrrHqJzto9m7SyiRzarwkqPcCSsKR2EB1AyqJfe8L8tN" && + VENUS="QmU5kp3BH3B8tnWUU2Pikdb2maksBNkb92FHRr56hyghh4" && + echo "added $IGNORE mountdir/planets/.ipfsignore" > expected && + echo "added $MARS mountdir/planets/mars.txt" >> expected && + echo "added $VENUS mountdir/planets/venus.txt" >> expected && + echo "added $PLANETS mountdir/planets" >> expected && + test_cmp expected actual +' + +test_expect_success "'ipfs add -r' respects .ipfsignore in active folder" ' + echo "Hello Mars!" >mountdir/planets/mars.txt && + echo "Hello Venus!" >mountdir/planets/venus.txt && + echo "ignore*" >mountdir/planets/.ipfsignore && + echo "this file should be ignored" >mountdir/planets/ignore_me.txt && + ipfs add -r mountdir/planets >actual + PLANETS="QmWSgS32xQEcXMeqd3YPJLrNBLSdsfYCep2U7CFkyrjXwY" && + MARS="QmPrrHqJzto9m7SyiRzarwkqPcCSsKR2EB1AyqJfe8L8tN" && + VENUS="QmU5kp3BH3B8tnWUU2Pikdb2maksBNkb92FHRr56hyghh4" && + echo "added $MARS mountdir/planets/mars.txt" >expected && + echo "added $VENUS mountdir/planets/venus.txt" >>expected && + echo "added $PLANETS mountdir/planets" >>expected && + test_cmp expected actual +' + +test_expect_success "'ipfs add -r' respects root .ipfsignore files" ' + mkdir -p mountdir/planets/moons && + echo "mars*" >mountdir/.ipfsignore && + echo "ignore*" >mountdir/planets/.ipfsignore && + echo "europa*" >mountdir/planets/moons/.ipfsignore && + echo "Hello Mars!" >mountdir/planets/mars.txt && + echo "Hello Venus!" >mountdir/planets/venus.txt && + echo "Hello IO!" >mountdir/planets/moons/io.txt && + echo "Hello Europa!" >mountdir/planets/moons/europa.txt && + echo "this file should be ignored" >mountdir/planets/ignore_me.txt && + ipfs add -r mountdir/planets > recursive_ignore && + echo "added QmTXxVggAJ1Jh3ZZgDteSMcrxX9T6VgxdxUisKmZMy2qdT mountdir/planets/moons/io.txt" > expected + echo "added Qmd9A64XXUgHqUkkxNiZww2C2yxm3uPbrfvvLfCujDFFGq mountdir/planets/moons" >> expected + echo "added QmU5kp3BH3B8tnWUU2Pikdb2maksBNkb92FHRr56hyghh4 mountdir/planets/venus.txt" >> expected + echo "added Qmc5Lpt1LczLsbvcx4AkX9YMJbfRYfwVyHufgYpP41SJBB mountdir/planets" >> expected + test_cmp expected recursive_ignore + #cat recursive_ignore +' test_expect_success "ipfs cat accept many hashes from stdin" ' { echo "$MARS"; echo "$VENUS"; } | ipfs cat >actual