diff --git a/.gitignore b/.gitignore index 1705cd8..afc7549 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ vendor coverage.out test.sh +tmp* diff --git a/.travis.yml b/.travis.yml index 40efd9d..370d5a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: go go: - - 1.11 - 1.12 - 1.13 @@ -15,7 +14,6 @@ env: before_script: - go get -u github.com/mattn/goveralls - cd $HOME - - mkdir -p $HOME/gopath/src/github.com/chrislusf/seaweedfs - mkdir -p /tmp/sw/master/metadata - mkdir -p /tmp/sw/volume1/data - mkdir -p /tmp/sw/filer diff --git a/filer.go b/filer.go index c62d477..8243710 100644 --- a/filer.go +++ b/filer.go @@ -42,10 +42,6 @@ func newFiler(u string, client *httpClient) (f *Filer, err error) { return } -var dirHeader = map[string]string{ - "Accept": "application/json", -} - // Close underlying daemons. func (f *Filer) Close() (err error) { if f.client != nil { diff --git a/seaweed.go b/seaweed.go index fdba0f3..c2c9e47 100644 --- a/seaweed.go +++ b/seaweed.go @@ -209,7 +209,7 @@ func (c *Seaweed) LookupServerByFileID(fileID string, args url.Values, readonly if err == nil { if readonly { - server = lookup.VolumeLocations.RandomPickForRead().URL + server = lookup.VolumeLocations.RandomPickForRead().PublicURL } else { server = lookup.VolumeLocations.Head().URL } @@ -295,30 +295,31 @@ func (c *Seaweed) SubmitFilePart(f *FilePart, args url.Values) (result *SubmitRe } // Upload file by reader. -func (c *Seaweed) Upload(fileReader io.Reader, fileName string, size int64, collection, ttl string) (fp *FilePart, fileID string, err error) { +func (c *Seaweed) Upload(fileReader io.Reader, fileName string, size int64, collection, ttl string) (fp *FilePart, err error) { fp = NewFilePartFromReader(ioutil.NopCloser(fileReader), fileName, size) fp.Collection, fp.TTL = collection, ttl - _, fileID, err = c.UploadFilePart(fp) + _, err = c.UploadFilePart(fp) return } // UploadFile with full file dir/path. -func (c *Seaweed) UploadFile(filePath string, collection, ttl string) (cm *ChunkManifest, fp *FilePart, fileID string, err error) { +func (c *Seaweed) UploadFile(filePath string, collection, ttl string) (cm *ChunkManifest, fp *FilePart, err error) { fp, err = NewFilePart(filePath) if err == nil { fp.Collection, fp.TTL = collection, ttl - cm, fileID, err = c.UploadFilePart(fp) + cm, err = c.UploadFilePart(fp) _ = fp.Close() } return } // UploadFilePart uploads a file part. -func (c *Seaweed) UploadFilePart(f *FilePart) (cm *ChunkManifest, fileID string, err error) { +func (c *Seaweed) UploadFilePart(f *FilePart) (cm *ChunkManifest, err error) { if f.FileID == "" { - res, err := c.Assign(normalize(nil, f.Collection, f.TTL)) + var res *AssignResult + res, err = c.Assign(normalize(nil, f.Collection, f.TTL)) if err != nil { - return nil, "", err + return } f.Server, f.FileID = res.URL, res.FileID } @@ -345,7 +346,7 @@ func (c *Seaweed) UploadFilePart(f *FilePart) (cm *ChunkManifest, fileID string, _, id, count, e := c.uploadChunk(f, baseName+"_"+strconv.FormatInt(i+1, 10)) if e != nil { // delete all uploaded chunks _ = c.DeleteChunks(cm, normalize(nil, f.Collection, "")) - return nil, "", e + return nil, e } cm.Chunks[i] = &ChunkInfo{ @@ -360,16 +361,14 @@ func (c *Seaweed) UploadFilePart(f *FilePart) (cm *ChunkManifest, fileID string, } } else { args := normalize(nil, f.Collection, f.TTL) - args.Set("Content-Type", "multipart/form-data") if f.ModTime != 0 { args.Set("ts", strconv.FormatInt(f.ModTime, 10)) } - _, _, err = c.client.upload(encodeURI(*c.master, f.FileID, args), baseName, f.Reader, f.MimeType) - } + base := *c.master + base.Host = f.Server - if err == nil { - fileID = f.FileID + _, _, err = c.client.upload(encodeURI(base, f.FileID, args), baseName, f.Reader, f.MimeType) } return @@ -433,8 +432,7 @@ func (c *Seaweed) BatchUploadFileParts(files []*FilePart, collection string, ttl func (c *Seaweed) uploadTask(file *FilePart) *workerpool.Task { return workerpool.NewTask(context.Background(), func(ctx context.Context) (res interface{}, err error) { - fmt.Println("Uploading", file) - _, _, err = c.UploadFilePart(file) + _, err = c.UploadFilePart(file) return }) } @@ -444,7 +442,7 @@ func (c *Seaweed) Replace(fileID string, newContent io.Reader, fileName string, fp := NewFilePartFromReader(ioutil.NopCloser(newContent), fileName, size) fp.Collection, fp.TTL = collection, ttl fp.FileID = fileID - _, err = c.ReplaceFilePart(fp, deleteFirst) + err = c.ReplaceFilePart(fp, deleteFirst) return } @@ -453,19 +451,19 @@ func (c *Seaweed) ReplaceFile(fileID, localFilePath string, deleteFirst bool) (e fp, err := NewFilePart(localFilePath) if err == nil { fp.FileID = fileID - _, err = c.ReplaceFilePart(fp, deleteFirst) + err = c.ReplaceFilePart(fp, deleteFirst) _ = fp.Close() } return } // ReplaceFilePart replaces file part. -func (c *Seaweed) ReplaceFilePart(f *FilePart, deleteFirst bool) (fileID string, err error) { +func (c *Seaweed) ReplaceFilePart(f *FilePart, deleteFirst bool) (err error) { if deleteFirst && f.FileID != "" { _ = c.DeleteFile(f.FileID, nil) } - _, fileID, err = c.UploadFilePart(f) + _, err = c.UploadFilePart(f) return } @@ -507,7 +505,10 @@ func (c *Seaweed) uploadManifest(f *FilePart, manifest *ChunkManifest) (err erro } args.Set("cm", "true") - _, _, err = c.client.upload(encodeURI(*c.master, f.FileID, args), manifest.Name, bufReader, "application/json") + base := *c.master + base.Host = f.Server + + _, _, err = c.client.upload(encodeURI(base, f.FileID, args), manifest.Name, bufReader, "application/json") } return } diff --git a/seaweed_test.go b/seaweed_test.go index 69316fa..23ca36b 100644 --- a/seaweed_test.go +++ b/seaweed_test.go @@ -39,10 +39,10 @@ func init() { filer = []string{_filer} } - sw, _ = NewSeaweed(masterURL, filer, 2*1024*1024, &http.Client{Timeout: 5 * time.Minute}) + sw, _ = NewSeaweed(masterURL, filer, 8096, &http.Client{Timeout: 5 * time.Minute}) _ = sw.Close() - sw, _ = NewSeaweed(masterURL, filer, 2*1024*1024, &http.Client{Timeout: 5 * time.Minute}) + sw, _ = NewSeaweed(masterURL, filer, 8096, &http.Client{Timeout: 5 * time.Minute}) MediumFile = os.Getenv("GOSWFS_MEDIUM_FILE") SmallFile = os.Getenv("GOSWFS_SMALL_FILE") @@ -50,63 +50,79 @@ func init() { func TestUploadLookupserverReplaceDeleteFile(t *testing.T) { for i := 0; i < 2; i++ { - _, _, fID, err := sw.UploadFile(MediumFile, "", "") + _, fp, err := sw.UploadFile(MediumFile, "", "") require.Nil(t, err) - _, err = sw.LookupServerByFileID(fID, nil, true) + _, err = sw.LookupServerByFileID(fp.FileID, nil, true) require.Nil(t, err) - // - _, err = sw.LookupFileID(fID, nil, true) + // verify by downloading + downloaded := verifyDownloadFile(t, fp.FileID) + fh, err := os.Open(MediumFile) + require.Nil(t, err) + allContent, _ := ioutil.ReadAll(fh) + require.Nil(t, fh.Close()) + require.EqualValues(t, downloaded, allContent) + + // try to looking up + _, err = sw.LookupFileID(fp.FileID, nil, true) require.Nil(t, err) - // - require.Nil(t, sw.ReplaceFile(fID, SmallFile, false)) - _, err = sw.LookupFileID(fID, nil, true) + // try to replace with small file + require.Nil(t, sw.ReplaceFile(fp.FileID, SmallFile, false)) + _, err = sw.LookupFileID(fp.FileID, nil, true) require.Nil(t, err) - // - require.Nil(t, sw.ReplaceFile(fID, SmallFile, true)) - _, err = sw.LookupFileID(fID, nil, true) + // verify by downloading + downloaded = verifyDownloadFile(t, fp.FileID) + fh, err = os.Open(SmallFile) require.Nil(t, err) + allContent, _ = ioutil.ReadAll(fh) + require.Nil(t, fh.Close()) + require.EqualValues(t, downloaded, allContent) - // - require.Nil(t, sw.DeleteFile(fID, nil)) + // replace again but delete first + require.Nil(t, sw.ReplaceFile(fp.FileID, SmallFile, true)) + _, err = sw.LookupFileID(fp.FileID, nil, true) + require.Nil(t, err) - // test upload file - fh, err := os.Open(MediumFile) + // verify by downloading + downloaded = verifyDownloadFile(t, fp.FileID) + fh, err = os.Open(SmallFile) + require.Nil(t, err) + allContent, _ = ioutil.ReadAll(fh) + require.Nil(t, fh.Close()) + require.EqualValues(t, downloaded, allContent) + + // delete file + require.Nil(t, sw.DeleteFile(fp.FileID, nil)) + + // uploading with file reader + fh, err = os.Open(MediumFile) require.Nil(t, err) var size int64 fi, fiErr := fh.Stat() require.Nil(t, fiErr) size = fi.Size() - _, fID, err = sw.Upload(fh, "test.txt", size, "col", "") + fp, err = sw.Upload(fh, "test.txt", size, "col", "") require.Nil(t, err) require.Nil(t, fh.Close()) - // Replace with small file + // Replace with small file reader fs, err := os.Open(SmallFile) require.Nil(t, err) fi, fiErr = fs.Stat() require.Nil(t, fiErr) size = fi.Size() - require.Nil(t, sw.Replace(fID, fs, "ta.txt", size, "", "", false)) - require.Nil(t, sw.DeleteFile(fID, nil)) + require.Nil(t, sw.Replace(fp.FileID, fs, "ta.txt", size, "", "", false)) + require.Nil(t, sw.DeleteFile(fp.FileID, nil)) fs.Close() } } func TestBatchUploadFiles(t *testing.T) { - if MediumFile != "" && SmallFile != "" { - _, err := sw.BatchUploadFiles([]string{MediumFile, SmallFile}, "", "") - require.Nil(t, err) - } else if MediumFile != "" { - _, err := sw.BatchUploadFiles([]string{MediumFile, MediumFile}, "", "") - require.Nil(t, err) - } else if SmallFile != "" { - _, err := sw.BatchUploadFiles([]string{SmallFile, SmallFile}, "", "") - require.Nil(t, err) - } + _, err := sw.BatchUploadFiles([]string{MediumFile, SmallFile}, "", "") + require.Nil(t, err) } func TestLookup(t *testing.T) { @@ -129,25 +145,34 @@ func TestClusterStatus(t *testing.T) { require.Nil(t, err) } -func TestSubmit(t *testing.T) { - if SmallFile != "" { - _, err := sw.Submit(SmallFile, "", "") - require.Nil(t, err) - } +func TestDownloadFile(t *testing.T) { + result, err := sw.Submit(SmallFile, "", "") + require.Nil(t, err) + require.NotNil(t, result) + + // return fake error + _, err = sw.Download(result.FileID, nil, func(r io.Reader) error { + return fmt.Errorf("Fake error") + }) + require.NotNil(t, err) + + // verifying + verifyDownloadFile(t, result.FileID) } -func TestDownloadFile(t *testing.T) { - if SmallFile != "" { - _, err := sw.Download(SmallFile, nil, func(r io.Reader) error { - return fmt.Errorf("Fake error") - }) - require.NotNil(t, err) - } +func verifyDownloadFile(t *testing.T, fid string) (data []byte) { + _, err := sw.Download(fid, nil, func(r io.Reader) (err error) { + data, err = ioutil.ReadAll(r) + return + }) + require.Nil(t, err) + require.NotZero(t, len(data)) + return } func TestDeleteChunks(t *testing.T) { if MediumFile != "" { - cm, _, _, err := sw.UploadFile(MediumFile, "", "") + cm, _, err := sw.UploadFile(MediumFile, "", "") require.Nil(t, err) err = sw.DeleteChunks(cm, nil) @@ -171,7 +196,6 @@ func TestFiler(t *testing.T) { }) require.Nil(t, err) require.NotZero(t, buf.Len()) - t.Log(string(buf.Bytes())) // try to delete this file err = filer.Delete("/js/test.txt", nil)