diff --git a/torchvision/io/video.py b/torchvision/io/video.py
index 40d1cfeed85..f0a47d49faf 100644
--- a/torchvision/io/video.py
+++ b/torchvision/io/video.py
@@ -106,7 +106,7 @@ def _read_from_stream(
         )
 
     frames = {}
-    should_buffer = False
+    should_buffer = True
     max_buffer_size = 5
     if stream.type == "video":
         # DivX-style packed B-frames can have out-of-order pts (2 frames in a single pkt)