diff --git a/aphrodite/v1/core/sched/scheduler.py b/aphrodite/v1/core/sched/scheduler.py
index 5ce64861db..ea4733356a 100644
--- a/aphrodite/v1/core/sched/scheduler.py
+++ b/aphrodite/v1/core/sched/scheduler.py
@@ -254,6 +254,7 @@ def schedule(self) -> SchedulerOutput:
                             key=lambda r: (r.priority, r.arrival_time),
                         )
                         self.running.remove(preempted_req)
+                        scheduled_running_reqs.remove(preempted_req)
                     else:
                         preempted_req = self.running.pop()