Merge pull request #2046 from ipfs/fix/multipart

flatten multipart transfers

commands/cli/parse.go

@@ -44,7 +44,17 @@ func Parse(input []string, stdin *os.File, root *cmds.Command) (cmds.Request, *c
 		}
 	}
 
-	stringArgs, fileArgs, err := parseArgs(stringVals, stdin, cmd.Arguments, recursive, root)
+	// if '--hidden' is provided, enumerate hidden paths
+	hiddenOpt := req.Option("hidden")
+	hidden := false
+	if hiddenOpt != nil {
+		hidden, _, err = hiddenOpt.Bool()
+		if err != nil {
+			return req, nil, nil, u.ErrCast()
+		}
+	}
+
+	stringArgs, fileArgs, err := parseArgs(stringVals, stdin, cmd.Arguments, recursive, hidden, root)
 	if err != nil {
 		return req, cmd, path, err
 	}
@@ -221,7 +231,7 @@ func parseOpts(args []string, root *cmds.Command) (
 	return
 }
 
-func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive bool, root *cmds.Command) ([]string, []files.File, error) {
+func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive, hidden bool, root *cmds.Command) ([]string, []files.File, error) {
 	// ignore stdin on Windows
 	if runtime.GOOS == "windows" {
 		stdin = nil
@@ -306,7 +316,7 @@ func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursi
 				// treat stringArg values as file paths
 				fpath := inputs[0]
 				inputs = inputs[1:]
-				file, err := appendFile(fpath, argDef, recursive)
+				file, err := appendFile(fpath, argDef, recursive, hidden)
 				if err != nil {
 					return nil, nil, err
 				}
@@ -387,7 +397,7 @@ func appendStdinAsString(args []string, stdin *os.File) ([]string, *os.File, err
 const notRecursiveFmtStr = "'%s' is a directory, use the '-%s' flag to specify directories"
 const dirNotSupportedFmtStr = "Invalid path '%s', argument '%s' does not support directories"
 
-func appendFile(fpath string, argDef *cmds.Argument, recursive bool) (files.File, error) {
+func appendFile(fpath string, argDef *cmds.Argument, recursive, hidden bool) (files.File, error) {
 	fpath = filepath.ToSlash(filepath.Clean(fpath))
 
 	if fpath == "." {
@@ -412,7 +422,7 @@ func appendFile(fpath string, argDef *cmds.Argument, recursive bool) (files.File
 		}
 	}
 
-	return files.NewSerialFile(path.Base(fpath), fpath, stat)
+	return files.NewSerialFile(path.Base(fpath), fpath, hidden, stat)
 }
 
 // isTerminal returns true if stdin is a Stdin pipe (e.g. `cat file | ipfs`),

commands/files/file_test.go

@@ -20,36 +20,38 @@ func TestSliceFiles(t *testing.T) {
 	sf := NewSliceFile(name, name, files)
 
 	if !sf.IsDirectory() {
-		t.Error("SliceFile should always be a directory")
+		t.Fatal("SliceFile should always be a directory")
 	}
-	if n, err := sf.Read(buf); n > 0 || err != ErrNotReader {
-		t.Error("Shouldn't be able to call `Read` on a SliceFile")
+
+	if n, err := sf.Read(buf); n > 0 || err != io.EOF {
+		t.Fatal("Shouldn't be able to read data from a SliceFile")
 	}
+
 	if err := sf.Close(); err != ErrNotReader {
-		t.Error("Shouldn't be able to call `Close` on a SliceFile")
+		t.Fatal("Shouldn't be able to call `Close` on a SliceFile")
 	}
 
 	file, err := sf.NextFile()
 	if file == nil || err != nil {
-		t.Error("Expected a file and nil error")
+		t.Fatal("Expected a file and nil error")
 	}
 	read, err := file.Read(buf)
 	if read != 11 || err != nil {
-		t.Error("NextFile got a file in the wrong order")
+		t.Fatal("NextFile got a file in the wrong order")
 	}
 
 	file, err = sf.NextFile()
 	if file == nil || err != nil {
-		t.Error("Expected a file and nil error")
+		t.Fatal("Expected a file and nil error")
 	}
 	file, err = sf.NextFile()
 	if file == nil || err != nil {
-		t.Error("Expected a file and nil error")
+		t.Fatal("Expected a file and nil error")
 	}
 
 	file, err = sf.NextFile()
 	if file != nil || err != io.EOF {
-		t.Error("Expected a nil file and io.EOF")
+		t.Fatal("Expected a nil file and io.EOF")
 	}
 }
 
@@ -59,21 +61,21 @@ func TestReaderFiles(t *testing.T) {
 	buf := make([]byte, len(message))
 
 	if rf.IsDirectory() {
-		t.Error("ReaderFile should never be a directory")
+		t.Fatal("ReaderFile should never be a directory")
 	}
 	file, err := rf.NextFile()
 	if file != nil || err != ErrNotDirectory {
-		t.Error("Expected a nil file and ErrNotDirectory")
+		t.Fatal("Expected a nil file and ErrNotDirectory")
 	}
 
 	if n, err := rf.Read(buf); n == 0 || err != nil {
-		t.Error("Expected to be able to read")
+		t.Fatal("Expected to be able to read")
 	}
 	if err := rf.Close(); err != nil {
-		t.Error("Should be able to close")
+		t.Fatal("Should be able to close")
 	}
 	if n, err := rf.Read(buf); n != 0 || err != io.EOF {
-		t.Error("Expected EOF when reading after close")
+		t.Fatal("Expected EOF when reading after close")
 	}
 }
 
@@ -86,23 +88,19 @@ Some-Header: beep
 
 beep
 --Boundary!
-Content-Type: multipart/mixed; boundary=OtherBoundary
+Content-Type: application/x-directory
 Content-Disposition: file; filename="dir"
 
---OtherBoundary
-Content-Type: text/plain
-Content-Disposition: file; filename="some/file/path"
-
-test
---OtherBoundary
+--Boundary!
 Content-Type: text/plain
+Content-Disposition: file; filename="dir/nested"
 
-boop
---OtherBoundary
-Content-Type: text/plain
+some content
+--Boundary!
+Content-Type: application/symlink
+Content-Disposition: file; filename="dir/simlynk"
 
-bloop
---OtherBoundary--
+anotherfile
 --Boundary!--
 
 `
@@ -114,81 +112,92 @@ bloop
 	// test properties of a file created from the first part
 	part, err := mpReader.NextPart()
 	if part == nil || err != nil {
-		t.Error("Expected non-nil part, nil error")
+		t.Fatal("Expected non-nil part, nil error")
 	}
 	mpf, err := NewFileFromPart(part)
 	if mpf == nil || err != nil {
-		t.Error("Expected non-nil MultipartFile, nil error")
+		t.Fatal("Expected non-nil MultipartFile, nil error")
 	}
 	if mpf.IsDirectory() {
-		t.Error("Expected file to not be a directory")
+		t.Fatal("Expected file to not be a directory")
 	}
 	if mpf.FileName() != "name" {
-		t.Error("Expected filename to be \"name\"")
+		t.Fatal("Expected filename to be \"name\"")
 	}
 	if file, err := mpf.NextFile(); file != nil || err != ErrNotDirectory {
-		t.Error("Expected a nil file and ErrNotDirectory")
+		t.Fatal("Expected a nil file and ErrNotDirectory")
 	}
 	if n, err := mpf.Read(buf); n != 4 || err != nil {
-		t.Error("Expected to be able to read 4 bytes")
+		t.Fatal("Expected to be able to read 4 bytes")
 	}
 	if err := mpf.Close(); err != nil {
-		t.Error("Expected to be able to close file")
+		t.Fatal("Expected to be able to close file")
 	}
 
 	// test properties of file created from second part (directory)
 	part, err = mpReader.NextPart()
 	if part == nil || err != nil {
-		t.Error("Expected non-nil part, nil error")
+		t.Fatal("Expected non-nil part, nil error")
 	}
 	mpf, err = NewFileFromPart(part)
 	if mpf == nil || err != nil {
-		t.Error("Expected non-nil MultipartFile, nil error")
+		t.Fatal("Expected non-nil MultipartFile, nil error")
 	}
 	if !mpf.IsDirectory() {
-		t.Error("Expected file to be a directory")
+		t.Fatal("Expected file to be a directory")
 	}
 	if mpf.FileName() != "dir" {
-		t.Error("Expected filename to be \"dir\"")
+		t.Fatal("Expected filename to be \"dir\"")
 	}
 	if n, err := mpf.Read(buf); n > 0 || err != ErrNotReader {
-		t.Error("Shouldn't be able to call `Read` on a directory")
+		t.Fatal("Shouldn't be able to call `Read` on a directory")
 	}
 	if err := mpf.Close(); err != ErrNotReader {
-		t.Error("Shouldn't be able to call `Close` on a directory")
+		t.Fatal("Shouldn't be able to call `Close` on a directory")
 	}
 
-	// test properties of first child file
-	child, err := mpf.NextFile()
-	if child == nil || err != nil {
-		t.Error("Expected to be able to read a child file")
+	// test properties of file created from third part (nested file)
+	part, err = mpReader.NextPart()
+	if part == nil || err != nil {
+		t.Fatal("Expected non-nil part, nil error")
 	}
-	if child.IsDirectory() {
-		t.Error("Expected file to not be a directory")
+	mpf, err = NewFileFromPart(part)
+	if mpf == nil || err != nil {
+		t.Fatal("Expected non-nil MultipartFile, nil error")
 	}
-	if child.FileName() != "some/file/path" {
-		t.Error("Expected filename to be \"some/file/path\"")
+	if mpf.IsDirectory() {
+		t.Fatal("Expected file, got directory")
+	}
+	if mpf.FileName() != "dir/nested" {
+		t.Fatalf("Expected filename to be \"nested\", got %s", mpf.FileName())
+	}
+	if n, err := mpf.Read(buf); n != 12 || err != nil {
+		t.Fatalf("expected to be able to read 12 bytes from file: %s (got %d)", err, n)
+	}
+	if err := mpf.Close(); err != nil {
+		t.Fatal("should be able to close file: %s", err)
 	}
 
-	// test processing files out of order
-	child, err = mpf.NextFile()
-	if child == nil || err != nil {
-		t.Error("Expected to be able to read a child file")
+	// test properties of symlink created from fourth part (symlink)
+	part, err = mpReader.NextPart()
+	if part == nil || err != nil {
+		t.Fatal("Expected non-nil part, nil error")
 	}
-	child2, err := mpf.NextFile()
-	if child == nil || err != nil {
-		t.Error("Expected to be able to read a child file")
+	mpf, err = NewFileFromPart(part)
+	if mpf == nil || err != nil {
+		t.Fatal("Expected non-nil MultipartFile, nil error")
 	}
-	if n, err := child2.Read(buf); n != 5 || err != nil {
-		t.Error("Expected to be able to read")
+	if mpf.IsDirectory() {
+		t.Fatal("Expected file to be a symlink")
 	}
-	if n, err := child.Read(buf); n != 0 || err == nil {
-		t.Error("Expected to not be able to read after advancing NextFile() past this file")
+	if mpf.FileName() != "dir/simlynk" {
+		t.Fatal("Expected filename to be \"dir/simlynk\"")
 	}
-
-	// make sure the end is handled properly
-	child, err = mpf.NextFile()
-	if child != nil || err == nil {
-		t.Error("Expected NextFile to return (nil, EOF)")
+	slink, ok := mpf.(*Symlink)
+	if !ok {
+		t.Fatalf("expected file to be a symlink")
+	}
+	if slink.Target != "anotherfile" {
+		t.Fatal("expected link to point to anotherfile")
 	}
 }

commands/files/multipartfile.go

@@ -1,18 +1,19 @@
 package files
 
 import (
+	"io"
 	"io/ioutil"
 	"mime"
 	"mime/multipart"
-	"net/http"
 	"net/url"
 )
 
 const (
 	multipartFormdataType = "multipart/form-data"
 	multipartMixedType    = "multipart/mixed"
 
-	applicationSymlink = "application/symlink"
+	applicationDirectory = "application/x-directory"
+	applicationSymlink   = "application/symlink"
 
 	contentTypeHeader = "Content-Type"
 )
@@ -45,40 +46,33 @@ func NewFileFromPart(part *multipart.Part) (File, error) {
 		}, nil
 	}
 
-	var params map[string]string
 	var err error
-	f.Mediatype, params, err = mime.ParseMediaType(contentType)
+	f.Mediatype, _, err = mime.ParseMediaType(contentType)
 	if err != nil {
 		return nil, err
 	}
 
-	if f.IsDirectory() {
-		boundary, found := params["boundary"]
-		if !found {
-			return nil, http.ErrMissingBoundary
-		}
-
-		f.Reader = multipart.NewReader(part, boundary)
-	}
-
 	return f, nil
 }
 
 func (f *MultipartFile) IsDirectory() bool {
-	return f.Mediatype == multipartFormdataType || f.Mediatype == multipartMixedType
+	return f.Mediatype == multipartFormdataType || f.Mediatype == applicationDirectory
 }
 
 func (f *MultipartFile) NextFile() (File, error) {
 	if !f.IsDirectory() {
 		return nil, ErrNotDirectory
 	}
+	if f.Reader != nil {
+		part, err := f.Reader.NextPart()
+		if err != nil {
+			return nil, err
+		}
 
-	part, err := f.Reader.NextPart()
-	if err != nil {
-		return nil, err
+		return NewFileFromPart(part)
 	}
 
-	return NewFileFromPart(part)
+	return nil, io.EOF
 }
 
 func (f *MultipartFile) FileName() string {