diff --git a/CHANGELOG.md b/CHANGELOG.md
index 216f991e..58e9fe64 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## Unreleased
+
+- Fix pool timeout to account for the total time spent retrying. (#823)
+
 ## 1.0.0 (November 6th, 2023)
 
 From version 1.0 our async support is now optional, as the package has minimal dependencies by default.
diff --git a/httpcore/_async/connection_pool.py b/httpcore/_async/connection_pool.py
index ddc0510e..a8a3d00c 100644
--- a/httpcore/_async/connection_pool.py
+++ b/httpcore/_async/connection_pool.py
@@ -1,11 +1,12 @@
 import ssl
 import sys
+import time
 from types import TracebackType
 from typing import AsyncIterable, AsyncIterator, Iterable, List, Optional, Type
 
 from .._backends.auto import AutoBackend
 from .._backends.base import SOCKET_OPTION, AsyncNetworkBackend
-from .._exceptions import ConnectionNotAvailable, UnsupportedProtocol
+from .._exceptions import ConnectionNotAvailable, PoolTimeout, UnsupportedProtocol
 from .._models import Origin, Request, Response
 from .._synchronization import AsyncEvent, AsyncLock, AsyncShieldCancellation
 from .connection import AsyncHTTPConnection
@@ -220,6 +221,13 @@ async def handle_async_request(self, request: Request) -> Response:
             )
 
         status = RequestStatus(request)
+        timeouts = request.extensions.get("timeout", {})
+        timeout = timeouts.get("pool", None)
+
+        if timeout is not None:
+            deadline = time.monotonic() + timeout
+        else:
+            deadline = float("inf")
 
         async with self._pool_lock:
             self._requests.append(status)
@@ -227,8 +235,6 @@ async def handle_async_request(self, request: Request) -> Response:
             await self._attempt_to_acquire_connection(status)
 
         while True:
-            timeouts = request.extensions.get("timeout", {})
-            timeout = timeouts.get("pool", None)
             try:
                 connection = await status.wait_for_connection(timeout=timeout)
             except BaseException as exc:
@@ -263,6 +269,10 @@ async def handle_async_request(self, request: Request) -> Response:
             else:
                 break
 
+            timeout = deadline - time.monotonic()
+            if timeout < 0:
+                raise PoolTimeout  # pragma: nocover
+
         # When we return the response, we wrap the stream in a special class
         # that handles notifying the connection pool once the response
         # has been released.
diff --git a/httpcore/_sync/connection_pool.py b/httpcore/_sync/connection_pool.py
index dbcaff1f..b586abf1 100644
--- a/httpcore/_sync/connection_pool.py
+++ b/httpcore/_sync/connection_pool.py
@@ -1,11 +1,12 @@
 import ssl
 import sys
+import time
 from types import TracebackType
 from typing import Iterable, Iterator, Iterable, List, Optional, Type
 
 from .._backends.sync import SyncBackend
 from .._backends.base import SOCKET_OPTION, NetworkBackend
-from .._exceptions import ConnectionNotAvailable, UnsupportedProtocol
+from .._exceptions import ConnectionNotAvailable, PoolTimeout, UnsupportedProtocol
 from .._models import Origin, Request, Response
 from .._synchronization import Event, Lock, ShieldCancellation
 from .connection import HTTPConnection
@@ -220,6 +221,13 @@ def handle_request(self, request: Request) -> Response:
             )
 
         status = RequestStatus(request)
+        timeouts = request.extensions.get("timeout", {})
+        timeout = timeouts.get("pool", None)
+
+        if timeout is not None:
+            deadline = time.monotonic() + timeout
+        else:
+            deadline = float("inf")
 
         with self._pool_lock:
             self._requests.append(status)
@@ -227,8 +235,6 @@ def handle_request(self, request: Request) -> Response:
             self._attempt_to_acquire_connection(status)
 
         while True:
-            timeouts = request.extensions.get("timeout", {})
-            timeout = timeouts.get("pool", None)
             try:
                 connection = status.wait_for_connection(timeout=timeout)
             except BaseException as exc:
@@ -263,6 +269,10 @@ def handle_request(self, request: Request) -> Response:
             else:
                 break
 
+            timeout = deadline - time.monotonic()
+            if timeout < 0:
+                raise PoolTimeout  # pragma: nocover
+
         # When we return the response, we wrap the stream in a special class
         # that handles notifying the connection pool once the response
         # has been released.