Merge pull request #25 from wolfeidau/fix_large_reads_not_working

fix: due to buffering in the underlying stream reads are limited in size

.gitignore

@@ -1,5 +1,5 @@
 /bin
-/coverage.*
+coverage.*
 .envrc
 /vendor
 /*.sarif

.vscode/settings.json

@@ -2,5 +2,8 @@
     "go.testFlags": [
         "-v",
         "-coverpkg=github.com/wolfeidau/s3iofs"
-    ]
+    ],
+    "go.testEnvVars": {
+        "AWS_DEBUG": "true"
+    }
 }

Makefile

@@ -15,4 +15,6 @@ test:
 	@echo "--- test all the things"
 	@go test -coverprofile=coverage.txt ./...
 	@go tool cover -func=coverage.txt
+	@cd integration; go test -coverpkg=github.com/wolfeidau/s3iofs -coverprofile=coverage.txt ./...
+	@cd integration; go tool cover -func=coverage.txt	
 .PHONY: test

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
 	golang.org/x/sys v0.14.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -35,6 +35,7 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.25.3/go.mod h1:4EqRHDCKP78hq3zOnmFXu
 github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI=
 github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -50,6 +51,12 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
 github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
@@ -61,5 +68,6 @@ golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

go.work.sum

@@ -1,4 +1,5 @@
 github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=

integration/s3fs_test.go

@@ -15,9 +15,17 @@ import (
 )
 
 var (
+	oneMegabyte    = 1024 * 1024
+	twoMegabytes   = 1024 * 1024 * 2
+	threeMegabytes = 1024 * 1024 * 3
+
 	oneKilobyte = bytes.Repeat([]byte("a"), 1024)
 )
 
+func generateData(length int) []byte {
+	return bytes.Repeat([]byte("a"), length)
+}
+
 func TestList(t *testing.T) {
 	assert := require.New(t)
 
@@ -85,21 +93,51 @@ func TestSeek(t *testing.T) {
 
 	s3fs := s3iofs.NewWithClient(testBucketName, client)
 
-	f, err := s3fs.Open("test_seek.txt")
-	assert.NoError(err)
+	t.Run("seek to start", func(t *testing.T) {
+		f, err := s3fs.Open("test_seek.txt")
+		assert.NoError(err)
 
-	rdr, ok := f.(io.ReadSeekCloser)
-	assert.True(ok)
+		rdr, ok := f.(io.ReadSeekCloser)
+		assert.True(ok)
 
-	defer rdr.Close()
+		defer rdr.Close()
 
-	n, err := rdr.Seek(512, 0)
-	assert.NoError(err)
-	assert.Equal(int64(512), n)
+		n, err := rdr.Seek(512, io.SeekStart)
+		assert.NoError(err)
+		assert.Equal(int64(512), n)
 
-	buf, err := io.ReadAll(rdr)
-	assert.NoError(err)
-	assert.Len(buf, 512)
+		buf, err := io.ReadAll(rdr)
+		assert.NoError(err)
+		assert.Len(buf, 512)
+	})
+
+	t.Run("seek to end", func(t *testing.T) {
+		f, err := s3fs.Open("test_seek.txt")
+		assert.NoError(err)
+		defer f.Close()
+		rdr, ok := f.(io.ReadSeekCloser)
+		assert.True(ok)
+		defer rdr.Close()
+		n, err := rdr.Seek(-512, io.SeekEnd)
+		assert.NoError(err)
+		assert.Equal(int64(512), n)
+	})
+
+	t.Run("seek to current", func(t *testing.T) {
+		f, err := s3fs.Open("test_seek.txt")
+		assert.NoError(err)
+		defer f.Close()
+		rdr, ok := f.(io.ReadSeekCloser)
+		assert.True(ok)
+		defer rdr.Close()
+		n, err := rdr.Seek(512, io.SeekCurrent)
+		assert.NoError(err)
+		assert.Equal(int64(512), n)
+
+		n, err = rdr.Seek(512, io.SeekCurrent)
+		assert.NoError(err)
+		assert.Equal(int64(1024), n)
+	})
 }
 
 func TestReaderAt(t *testing.T) {
@@ -131,3 +169,71 @@ func TestReaderAt(t *testing.T) {
 	assert.NoError(err)
 	assert.Equal(0, n)
 }
+
+func TestReaderAtBig(t *testing.T) {
+	assert := require.New(t)
+
+	_, err := client.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(testBucketName),
+		Key:    aws.String("test_reader_at_big.txt"),
+		Body:   bytes.NewReader(generateData(threeMegabytes)),
+	})
+	assert.NoError(err)
+
+	s3fs := s3iofs.NewWithClient(testBucketName, client)
+
+	f, err := s3fs.Open("test_reader_at_big.txt")
+	assert.NoError(err)
+
+	defer f.Close()
+
+	rdr, ok := f.(io.ReaderAt)
+	assert.True(ok)
+
+	n, err := rdr.ReadAt(make([]byte, oneMegabyte), 0)
+	assert.NoError(err)
+	assert.Equal(oneMegabyte, n)
+
+	n, err = rdr.ReadAt(make([]byte, twoMegabytes), 0)
+	assert.NoError(err)
+	assert.Equal(twoMegabytes, n)
+}
+
+func TestReadFile(t *testing.T) {
+	assert := require.New(t)
+
+	_, err := client.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(testBucketName),
+		Key:    aws.String("test_read_big.txt"),
+		Body:   bytes.NewReader(generateData(threeMegabytes)),
+	})
+	assert.NoError(err)
+
+	s3fs := s3iofs.NewWithClient(testBucketName, client)
+
+	data, err := fs.ReadFile(s3fs, "test_read_big.txt")
+	assert.NoError(err)
+	assert.Len(data, threeMegabytes)
+}
+
+func TestReadBigEOF(t *testing.T) {
+	assert := require.New(t)
+
+	_, err := client.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(testBucketName),
+		Key:    aws.String("test_read_big_eof.txt"),
+		Body:   bytes.NewReader(generateData(oneMegabyte)),
+	})
+	assert.NoError(err)
+
+	s3fs := s3iofs.NewWithClient(testBucketName, client)
+
+	f, err := s3fs.Open("test_read_big_eof.txt")
+	assert.NoError(err)
+
+	defer f.Close()
+
+	n, err := io.ReadFull(f, make([]byte, twoMegabytes))
+	assert.ErrorIs(err, io.ErrUnexpectedEOF)
+	assert.Equal(oneMegabyte, n)
+}

integration/setup_test.go

@@ -61,7 +61,6 @@ func TestMain(m *testing.M) {
 	endpoint = fmt.Sprintf("http://%s", resource.GetHostPort("9000/tcp"))
 
 	if err := pool.Retry(func() error {
-
 		endpointURL, err := url.Parse(endpoint)
 		if err != nil {
 			log.Fatalf("failed to parse endpoint URL: %s", err)

s3.go

@@ -6,7 +6,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 )
 
-// S3API s3 calls used to build this library, this is used to enable testing
+// S3API s3 calls used to build this library, this is used to enable testing.
 type S3API interface {
 	GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
 	ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)