diff --git a/vllm/core/scheduler.py b/vllm/core/scheduler.py
index 399665082f838..0159053b4dc6a 100644
--- a/vllm/core/scheduler.py
+++ b/vllm/core/scheduler.py
@@ -905,7 +905,8 @@ def _schedule_chunked_prefill(self):
             blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
             blocks_to_copy=running_scheduled.blocks_to_copy +
             swapped_in.blocks_to_copy,
-            ignored_seq_groups=prefills.ignored_seq_groups,
+            ignored_seq_groups=prefills.ignored_seq_groups +
+            swapped_in.infeasible_seq_groups,
             num_lookahead_slots=running_scheduled.num_lookahead_slots,
             running_queue_size=len(self.running),
             preempted=(len(running_scheduled.preempted) +