diff --git a/gax/src/main/java/com/google/api/gax/rpc/ApiCallContext.java b/gax/src/main/java/com/google/api/gax/rpc/ApiCallContext.java
index 1115027d6..2336be0e7 100644
--- a/gax/src/main/java/com/google/api/gax/rpc/ApiCallContext.java
+++ b/gax/src/main/java/com/google/api/gax/rpc/ApiCallContext.java
@@ -63,9 +63,13 @@ public interface ApiCallContext extends RetryingContext {
    * Returns a new ApiCallContext with the given timeout set.
    *
    * <p>This sets the maximum amount of time a single unary RPC attempt can take. If retries are
-   * enabled, then this can take much longer. Unlike a deadline, timeouts are relative durations
-   * that are measure from the beginning of each RPC attempt. Please note that this will limit the
-   * duration of a server streaming RPC as well.
+   * enabled, then this can take much longer, as each RPC attempt will have the same constant
+   * timeout. Unlike a deadline, timeouts are relative durations that are measure from the beginning
+   * of each RPC attempt. Please note that this limits the duration of a server streaming RPC as
+   * well.
+   *
+   * <p>If a method has default {@link com.google.api.gax.retrying.RetrySettings}, the max attempts
+   * and/or total timeout is still respected when scheduling each RPC attempt.
    */
   ApiCallContext withTimeout(@Nullable Duration timeout);
 
diff --git a/gax/src/main/java/com/google/api/gax/rpc/AttemptCallable.java b/gax/src/main/java/com/google/api/gax/rpc/AttemptCallable.java
index dbd9c995c..e053d5583 100644
--- a/gax/src/main/java/com/google/api/gax/rpc/AttemptCallable.java
+++ b/gax/src/main/java/com/google/api/gax/rpc/AttemptCallable.java
@@ -69,8 +69,9 @@ public ResponseT call() {
     ApiCallContext callContext = originalCallContext;
 
     try {
+      // Set the RPC timeout if the caller did not provide their own.
       Duration rpcTimeout = externalFuture.getAttemptSettings().getRpcTimeout();
-      if (!rpcTimeout.isZero()) {
+      if (!rpcTimeout.isZero() && callContext.getTimeout() == null) {
         callContext = callContext.withTimeout(rpcTimeout);
       }
 
diff --git a/gax/src/test/java/com/google/api/gax/rpc/AttemptCallableTest.java b/gax/src/test/java/com/google/api/gax/rpc/AttemptCallableTest.java
index cb5133620..3b16b568d 100644
--- a/gax/src/test/java/com/google/api/gax/rpc/AttemptCallableTest.java
+++ b/gax/src/test/java/com/google/api/gax/rpc/AttemptCallableTest.java
@@ -108,6 +108,9 @@ public void testRpcTimeoutIsNotErased() {
     Duration callerTimeout = Duration.ofMillis(10);
     ApiCallContext callerCallContext = FakeCallContext.createDefault().withTimeout(callerTimeout);
 
+    Duration timeout = Duration.ofMillis(5);
+    currentAttemptSettings = currentAttemptSettings.toBuilder().setRpcTimeout(timeout).build();
+
     AttemptCallable<String, String> callable =
         new AttemptCallable<>(mockInnerCallable, "fake-request", callerCallContext);
     callable.setExternalFuture(mockExternalFuture);