diff --git a/commands/cli/parse.go b/commands/cli/parse.go index 6de79e6035b..4df4d20614b 100644 --- a/commands/cli/parse.go +++ b/commands/cli/parse.go @@ -43,7 +43,17 @@ func Parse(input []string, stdin *os.File, root *cmds.Command) (cmds.Request, *c } } - stringArgs, fileArgs, err := parseArgs(stringVals, stdin, cmd.Arguments, recursive, root) + // if '--hidden' is provided, enumerate hidden paths + hiddenOpt := req.Option("hidden") + hidden := false + if hiddenOpt != nil { + hidden, _, err = hiddenOpt.Bool() + if err != nil { + return req, nil, nil, u.ErrCast() + } + } + + stringArgs, fileArgs, err := parseArgs(stringVals, stdin, cmd.Arguments, recursive, hidden, root) if err != nil { return req, cmd, path, err } @@ -213,7 +223,7 @@ func parseOpts(args []string, root *cmds.Command) ( return } -func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive bool, root *cmds.Command) ([]string, []files.File, error) { +func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive, hidden bool, root *cmds.Command) ([]string, []files.File, error) { // ignore stdin on Windows if runtime.GOOS == "windows" { stdin = nil @@ -298,7 +308,7 @@ func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursi // treat stringArg values as file paths fpath := inputs[0] inputs = inputs[1:] - file, err := appendFile(fpath, argDef, recursive) + file, err := appendFile(fpath, argDef, recursive, hidden) if err != nil { return nil, nil, err } @@ -379,7 +389,7 @@ func appendStdinAsString(args []string, stdin *os.File) ([]string, *os.File, err const notRecursiveFmtStr = "'%s' is a directory, use the '-%s' flag to specify directories" const dirNotSupportedFmtStr = "Invalid path '%s', argument '%s' does not support directories" -func appendFile(fpath string, argDef *cmds.Argument, recursive bool) (files.File, error) { +func appendFile(fpath string, argDef *cmds.Argument, recursive, hidden bool) (files.File, error) { if fpath == "." { cwd, err := os.Getwd() @@ -403,7 +413,7 @@ func appendFile(fpath string, argDef *cmds.Argument, recursive bool) (files.File } } - return files.NewSerialFile(path.Base(fpath), fpath, stat) + return files.NewSerialFile(path.Base(fpath), fpath, hidden, stat) } // isTerminal returns true if stdin is a Stdin pipe (e.g. `cat file | ipfs`), diff --git a/commands/files/multipartfile.go b/commands/files/multipartfile.go index 7cd4930a8ed..b71dd7fe600 100644 --- a/commands/files/multipartfile.go +++ b/commands/files/multipartfile.go @@ -1,10 +1,10 @@ package files import ( + "io" "io/ioutil" "mime" "mime/multipart" - "net/http" "net/url" ) @@ -12,7 +12,8 @@ const ( multipartFormdataType = "multipart/form-data" multipartMixedType = "multipart/mixed" - applicationSymlink = "application/symlink" + applicationDirectory = "application/x-directory" + applicationSymlink = "application/symlink" contentTypeHeader = "Content-Type" ) @@ -45,40 +46,33 @@ func NewFileFromPart(part *multipart.Part) (File, error) { }, nil } - var params map[string]string var err error - f.Mediatype, params, err = mime.ParseMediaType(contentType) + f.Mediatype, _, err = mime.ParseMediaType(contentType) if err != nil { return nil, err } - if f.IsDirectory() { - boundary, found := params["boundary"] - if !found { - return nil, http.ErrMissingBoundary - } - - f.Reader = multipart.NewReader(part, boundary) - } - return f, nil } func (f *MultipartFile) IsDirectory() bool { - return f.Mediatype == multipartFormdataType || f.Mediatype == multipartMixedType + return f.Mediatype == multipartFormdataType || f.Mediatype == applicationDirectory } func (f *MultipartFile) NextFile() (File, error) { if !f.IsDirectory() { return nil, ErrNotDirectory } + if f.Reader != nil { + part, err := f.Reader.NextPart() + if err != nil { + return nil, err + } - part, err := f.Reader.NextPart() - if err != nil { - return nil, err + return NewFileFromPart(part) } - return NewFileFromPart(part) + return nil, io.EOF } func (f *MultipartFile) FileName() string { diff --git a/commands/files/serialfile.go b/commands/files/serialfile.go index b43869a4d47..7d2fcd91b69 100644 --- a/commands/files/serialfile.go +++ b/commands/files/serialfile.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "os" fp "path/filepath" + "strings" "syscall" ) @@ -18,9 +19,10 @@ type serialFile struct { files []os.FileInfo stat os.FileInfo current *File + hidden bool } -func NewSerialFile(name, path string, stat os.FileInfo) (File, error) { +func NewSerialFile(name, path string, hidden bool, stat os.FileInfo) (File, error) { switch mode := stat.Mode(); { case mode.IsRegular(): file, err := os.Open(path) @@ -35,7 +37,7 @@ func NewSerialFile(name, path string, stat os.FileInfo) (File, error) { if err != nil { return nil, err } - return &serialFile{name, path, contents, stat, nil}, nil + return &serialFile{name, path, contents, stat, nil, hidden}, nil case mode&os.ModeSymlink != 0: target, err := os.Readlink(path) if err != nil { @@ -68,6 +70,15 @@ func (f *serialFile) NextFile() (File, error) { stat := f.files[0] f.files = f.files[1:] + for !f.hidden && strings.HasPrefix(stat.Name(), ".") { + if len(f.files) == 0 { + return nil, io.EOF + } + + stat = f.files[0] + f.files = f.files[1:] + } + // open the next file fileName := fp.Join(f.name, stat.Name()) filePath := fp.Join(f.path, stat.Name()) @@ -75,7 +86,7 @@ func (f *serialFile) NextFile() (File, error) { // recursively call the constructor on the next file // if it's a regular file, we will open it as a ReaderFile // if it's a directory, files in it will be opened serially - sf, err := NewSerialFile(fileName, filePath, stat) + sf, err := NewSerialFile(fileName, filePath, f.hidden, stat) if err != nil { return nil, err } @@ -94,7 +105,7 @@ func (f *serialFile) FullPath() string { } func (f *serialFile) Read(p []byte) (int, error) { - return 0, ErrNotReader + return 0, io.EOF } func (f *serialFile) Close() error { diff --git a/commands/files/slicefile.go b/commands/files/slicefile.go index b705151f152..8d18dcaa372 100644 --- a/commands/files/slicefile.go +++ b/commands/files/slicefile.go @@ -41,7 +41,7 @@ func (f *SliceFile) FullPath() string { } func (f *SliceFile) Read(p []byte) (int, error) { - return 0, ErrNotReader + return 0, io.EOF } func (f *SliceFile) Close() error { diff --git a/commands/http/client.go b/commands/http/client.go index 186152adf10..3da268ffee9 100644 --- a/commands/http/client.go +++ b/commands/http/client.go @@ -13,7 +13,6 @@ import ( "strings" cmds "github.com/ipfs/go-ipfs/commands" - path "github.com/ipfs/go-ipfs/path" config "github.com/ipfs/go-ipfs/repo/config" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" @@ -86,8 +85,8 @@ func (c *client) Send(req cmds.Request) (cmds.Response, error) { reader = strings.NewReader("") } - pth := path.Join(req.Path()) - url := fmt.Sprintf(ApiUrlFormat, c.serverAddress, ApiPath, pth, query) + path := strings.Join(req.Path(), "/") + url := fmt.Sprintf(ApiUrlFormat, c.serverAddress, ApiPath, path, query) httpReq, err := http.NewRequest("POST", url, reader) if err != nil { diff --git a/commands/http/multifilereader.go b/commands/http/multifilereader.go index 6378ab49385..4a564176a3e 100644 --- a/commands/http/multifilereader.go +++ b/commands/http/multifilereader.go @@ -17,7 +17,7 @@ import ( type MultiFileReader struct { io.Reader - files files.File + files []files.File currentFile io.Reader buf bytes.Buffer mpWriter *multipart.Writer @@ -34,7 +34,7 @@ type MultiFileReader struct { // if `form` is false, the Content-Type will be 'multipart/mixed'. func NewMultiFileReader(file files.File, form bool) *MultiFileReader { mfr := &MultiFileReader{ - files: file, + files: []files.File{file}, form: form, mutex: &sync.Mutex{}, } @@ -54,34 +54,41 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // if the current file isn't set, advance to the next file if mfr.currentFile == nil { - file, err := mfr.files.NextFile() - if err == io.EOF { - mfr.mpWriter.Close() - mfr.closed = true - } else if err != nil { - return 0, err + var file files.File + for file == nil { + if len(mfr.files) == 0 { + mfr.mpWriter.Close() + mfr.closed = true + return mfr.buf.Read(buf) + } + + nextfile, err := mfr.files[len(mfr.files)-1].NextFile() + if err == io.EOF { + mfr.files = mfr.files[:len(mfr.files)-1] + continue + } else if err != nil { + return 0, err + } + + file = nextfile } // handle starting a new file part if !mfr.closed { var contentType string - if s, ok := file.(*files.Symlink); ok { - mfr.currentFile = s - + if _, ok := file.(*files.Symlink); ok { contentType = "application/symlink" } else if file.IsDirectory() { - // if file is a directory, create a multifilereader from it - // (using 'multipart/mixed') - nmfr := NewMultiFileReader(file, false) - mfr.currentFile = nmfr - contentType = fmt.Sprintf("multipart/mixed; boundary=%s", nmfr.Boundary()) + mfr.files = append(mfr.files, file) + contentType = "application/x-directory" } else { // otherwise, use the file as a reader to read its contents - mfr.currentFile = file contentType = "application/octet-stream" } + mfr.currentFile = file + // write the boundary and headers header := make(textproto.MIMEHeader) filename := url.QueryEscape(file.FileName()) diff --git a/core/commands/add.go b/core/commands/add.go index ef7c9748af0..e01645f6093 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -2,7 +2,6 @@ package commands import ( "fmt" - "io" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb" "github.com/ipfs/go-ipfs/core/coreunix" @@ -145,26 +144,8 @@ remains to be implemented. fileAdder.Pin = dopin fileAdder.Silent = silent - // addAllFiles loops over a convenience slice file to - // add each file individually. e.g. 'ipfs add a b c' - addAllFiles := func(sliceFile files.File) error { - for { - file, err := sliceFile.NextFile() - if err != nil && err != io.EOF { - return err - } - if file == nil { - return nil // done - } - - if err := fileAdder.AddFile(file); err != nil { - return err - } - } - } - addAllAndPin := func(f files.File) error { - if err := addAllFiles(f); err != nil { + if err := fileAdder.AddFile(f); err != nil { return err } diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 64bb6ad3062..5d798d588c2 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -256,7 +256,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { return "", err } - f, err := files.NewSerialFile(root, root, stat) + f, err := files.NewSerialFile(root, root, false, stat) if err != nil { return "", err } @@ -355,7 +355,7 @@ func (adder *Adder) addFile(file files.File) error { switch { case files.IsHidden(file) && !adder.Hidden: - log.Debugf("%s is hidden, skipping", file.FileName()) + log.Infof("%s is hidden, skipping", file.FileName()) return &hiddenFileError{file.FileName()} case file.IsDirectory(): return adder.addDir(file) diff --git a/exchange/bitswap/workers.go b/exchange/bitswap/workers.go index 04d9fc2d29f..0c8b8de5d1e 100644 --- a/exchange/bitswap/workers.go +++ b/exchange/bitswap/workers.go @@ -89,7 +89,7 @@ func (bs *Bitswap) provideWorker(px process.Process) { defer cancel() if err := bs.network.Provide(ctx, k); err != nil { - log.Error(err) + log.Warning(err) } } diff --git a/importer/importer.go b/importer/importer.go index b16b5b05bd0..74436e6236b 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -29,7 +29,7 @@ func BuildDagFromFile(fpath string, ds dag.DAGService) (*dag.Node, error) { return nil, fmt.Errorf("`%s` is a directory", fpath) } - f, err := files.NewSerialFile(fpath, fpath, stat) + f, err := files.NewSerialFile(fpath, fpath, false, stat) if err != nil { return nil, err } diff --git a/mfs/ops.go b/mfs/ops.go index ebb1932edeb..fc36b2256d3 100644 --- a/mfs/ops.go +++ b/mfs/ops.go @@ -102,7 +102,7 @@ func PutNode(r *Root, path string, nd *dag.Node) error { // intermediary directories as needed if 'parents' is set to true func Mkdir(r *Root, pth string, parents bool) error { if pth == "" { - panic("empty path") + return nil } parts := path.SplitList(pth) if parts[0] == "" {