[core] Erases PopWorkerRequest on registration timeout. (#48858)

This fixes a bug from #47694. When a PopWorkerRequest is time out, we
should erase it from bookkeeping and invoke the callback with a
`WorkerPendingRegistration` status. However only the latter is done and
we forgot to erase, causing segfaults on a second call.

Signed-off-by: Ruiyang Wang <rywang014@gmail.com>

src/ray/raylet/worker_pool.cc

@@ -593,7 +593,8 @@ void WorkerPool::MonitorPopWorkerRequestForRegistration(
     auto &requests = state.pending_registration_requests;
     auto it = std::find(requests.begin(), requests.end(), pop_worker_request);
     if (it != requests.end()) {
-      // Fail the task...
+      // Pop and fail the task...
+      requests.erase(it);
       PopWorkerStatus status = PopWorkerStatus::WorkerPendingRegistration;
       PopWorkerCallbackAsync(pop_worker_request->callback, nullptr, status);
     }

src/ray/raylet/worker_pool_test.cc

@@ -1928,6 +1928,19 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerStatus) {
   worker_pool_->ClearProcesses();
 }
 
+TEST_F(WorkerPoolDriverRegisteredTest, WorkerPendingRegistrationErasesRequest) {
+  std::shared_ptr<WorkerInterface> popped_worker;
+  PopWorkerStatus status;
+  auto task_spec = ExampleTaskSpec();
+  // Create a task without push worker. It should time out (WorkerPendingRegistration).
+  popped_worker = worker_pool_->PopWorkerSync(task_spec, false, &status);
+  ASSERT_EQ(popped_worker, nullptr);
+  ASSERT_EQ(status, PopWorkerStatus::WorkerPendingRegistration);
+  // The request should be erased.
+  ASSERT_EQ(worker_pool_->NumPendingRegistrationRequests(), 0);
+  worker_pool_->ClearProcesses();
+}
+
 TEST_F(WorkerPoolDriverRegisteredTest, TestIOWorkerFailureAndSpawn) {
   std::unordered_set<std::shared_ptr<WorkerInterface>> spill_worker_set;
   auto spill_worker_callback =