Optimize for files larger than 16MiB

2 files changed, 36 insertions, 2 deletions

diff --git a/fs.go b/fs.go
index 4ccfd45..864570b 100644
--- a/fs.go
+++ b/fs.go
@@ -230,12 +230,28 @@ func (fs *Fs) newObjectWriter(key string) (io.WriteCloser, context.CancelFunc, <
 
 	errc := make(chan error, 1)
 	go func() {
-		b, err := io.ReadAll(r)
+		maxAtomicSize := int64(16 * (1 << 20))
+		lr := io.LimitReader(r, maxAtomicSize)
+		b, err := io.ReadAll(lr)
 		if err != nil {
 			errc <- err
 			return
 		}
-		_, err = fs.client.PutObject(ctx, fs.bucket, key, bytes.NewReader(b), int64(len(b)), minio.PutObjectOptions{})
+		if ctx.Err() != nil {
+			errc <- ctx.Err()
+			return
+		}
+		var rd io.Reader
+		var size int64
+		if int64(len(b)) == maxAtomicSize {
+			rd = io.MultiReader(bytes.NewReader(b), r)
+			size = -1
+		} else {
+			rd = bytes.NewReader(b)
+			size = int64(len(b))
+		}
+
+		_, err = fs.client.PutObject(ctx, fs.bucket, key, rd, size, minio.PutObjectOptions{})
 		errc <- fromErrorResponse(err)
 	}()
 
diff --git a/fs_test.go b/fs_test.go
index 550c199..7e9765c 100644
--- a/fs_test.go
+++ b/fs_test.go
@@ -1,6 +1,7 @@
 package s3
 
 import (
+	cryptorand "crypto/rand"
 	"errors"
 	"io"
 	"os"
@@ -98,6 +99,23 @@ func TestFsWriteError(t *testing.T) {
 	is.Equal(err, os.ErrNotExist)
 }
 
+func TestFsWrite17MB(t *testing.T) {
+	is := is.New(t)
+	afs, cleanup := newTestAfs()
+	defer cleanup()
+
+	f, err := afs.Fs.Create("dir/file")
+	is.NoErr(err)
+	r := io.LimitReader(cryptorand.Reader, 17*(1<<20))
+	_, err = io.Copy(f, r)
+	is.NoErr(err)
+	err = f.Close()
+	is.NoErr(err)
+	info, err := afs.Stat("dir/file")
+	is.NoErr(err)
+	is.Equal(info.Size(), int64(17*(1<<20)))
+}
+
 func TestFsStat(t *testing.T) {
 	is := is.New(t)
 	afs, cleanup := newTestAfs()