Return S3 reader wrapped by BufferedReader

pfnet · Dec 26, 2021 · d5f1f6c · d5f1f6c
1 parent 69f87d8
commit d5f1f6c
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/pfio/v2/s3.py b/pfio/v2/s3.py
@@ -10,6 +10,9 @@
 from .fs import FS, FileStat
 
 
+DEFAULT_BUFFER_SIZE = 32 * 1024 * 1024
+
+
 def _normalize_key(key: str) -> str:
     key = os.path.normpath(key)
     if key.startswith("/"):
@@ -367,9 +370,7 @@ def open(self, path, mode='r', **kwargs):
         if 'r' in mode:
             obj = _ObjectReader(self.client, self.bucket, path, mode, kwargs)
             if 'b' in mode:
-                # TODO: BufferedIOBase requires readinto() implemeted
-                # obj = io.BufferedReader(obj)
-                pass
+                obj = io.BufferedReader(obj, buffer_size=DEFAULT_BUFFER_SIZE)
             else:
                 obj = io.TextIOWrapper(obj)