diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md
index 254e7312b53..a30103ce3a6 100644
--- a/api-extractor/report/hls.js.api.md
+++ b/api-extractor/report/hls.js.api.md
@@ -291,7 +291,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP
     // Warning: (ae-forgotten-export) The symbol "RemuxedTrack" needs to be exported by the entry point hls.d.ts
     //
     // (undocumented)
-    protected bufferFragmentData(data: RemuxedTrack, frag: Fragment, part: Part | null, chunkMeta: ChunkMetadata): void;
+    protected bufferFragmentData(data: RemuxedTrack, frag: Fragment, part: Part | null, chunkMeta: ChunkMetadata, noBacktracking?: boolean): void;
     // (undocumented)
     protected clearTrackerIfNeeded(frag: Fragment): void;
     // (undocumented)
diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts
index 4034f9589be..5c82fc6f62a 100644
--- a/src/controller/base-stream-controller.ts
+++ b/src/controller/base-stream-controller.ts
@@ -873,7 +873,8 @@ export default class BaseStreamController
     data: RemuxedTrack,
     frag: Fragment,
     part: Part | null,
-    chunkMeta: ChunkMetadata
+    chunkMeta: ChunkMetadata,
+    noBacktracking?: boolean
   ) {
     if (!data || this.state !== State.PARSING) {
       return;
@@ -901,6 +902,9 @@ export default class BaseStreamController
     this.hls.trigger(Events.BUFFER_APPENDING, segment);
 
     if (data.dropped && data.independent && !part) {
+      if (noBacktracking) {
+        return;
+      }
       // Clear buffer so that we reload previous segments sequentially if required
       this.flushBufferGap(frag);
     }
diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts
index 74ef5260d63..2386fc9ec4b 100644
--- a/src/controller/stream-controller.ts
+++ b/src/controller/stream-controller.ts
@@ -1175,7 +1175,13 @@ export default class StreamController
         if (this.backtrackFragment) {
           this.backtrackFragment = frag;
         }
-        this.bufferFragmentData(video, frag, part, chunkMeta);
+        this.bufferFragmentData(
+          video,
+          frag,
+          part,
+          chunkMeta,
+          isFirstFragment || isFirstInDiscontinuity
+        );
       } else if (isFirstFragment || isFirstInDiscontinuity) {
         // Mark segment with a gap to avoid loop loading
         frag.gap = true;