[credentialhelper] Ignore failing to provide request #17666
Closed
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This fixes a race condition where writing to stdin of the subprocess fails because the subprocess terminated quickly (e.g., because it did not need the request to provide credentials, or some other error occurred in the helper before it reads from stdin). ``` Starting local Bazel server and connecting to it... INFO: Writing tracer profile to '/private/var/tmp/_bazel_yannic/bb6c3b02ccc98aaca966df445b5024eb/command.profile.gz' INFO: Invocation ID: 86e227bc-187e-4d28-8f04-40b1ce9a36a4 ERROR: java.io.IOException: Stream closed at java.base/java.lang.ProcessBuilder$NullOutputStream.write(Unknown Source) at java.base/java.io.OutputStream.write(Unknown Source) at java.base/java.io.BufferedOutputStream.flushBuffer(Unknown Source) at java.base/java.io.BufferedOutputStream.flush(Unknown Source) at java.base/java.io.FilterOutputStream.close(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.implClose(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.close(Unknown Source) at java.base/java.io.OutputStreamWriter.close(Unknown Source) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelper.getCredentials(CredentialHelper.java:82) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getCredentialsFromHelper(CredentialHelperCredentials.java:108) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2413) at java.base/java.util.concurrent.ConcurrentHashMap.compute(Unknown Source) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2411) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2394) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getRequestMetadata(CredentialHelperCredentials.java:80) at com.google.auth.Credentials.blockingGetToCallback(Credentials.java:112) at com.google.auth.Credentials$1.run(Credentials.java:98) at io.grpc.stub.ClientCalls$ThreadlessExecutor.runQuietly(ClientCalls.java:771) at io.grpc.stub.ClientCalls$ThreadlessExecutor.waitAndDrain(ClientCalls.java:754) at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:157) at build.bazel.remote.execution.v2.CapabilitiesGrpc$CapabilitiesBlockingStub.getCapabilities(CapabilitiesGrpc.java:218) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$0(RemoteServerCapabilities.java:84) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannelBlocking$2(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannel$4(ReferenceCountedChannel.java:108) at io.reactivex.rxjava3.internal.operators.single.SingleUsing.subscribeActual(SingleUsing.java:59) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:112) at io.reactivex.rxjava3.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:65) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose$DoOnDisposeObserver.onSuccess(SingleDoOnDispose.java:84) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError$DoOnError.onSuccess(SingleDoOnError.java:52) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle$SingleElementObserver.onComplete(ObservableSingleSingle.java:110) at io.reactivex.rxjava3.internal.observers.DeferredScalarDisposable.complete(DeferredScalarDisposable.java:85) at io.reactivex.rxjava3.subjects.AsyncSubject.subscribeActual(AsyncSubject.java:233) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle.subscribeActual(ObservableSingleSingle.java:36) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError.subscribeActual(SingleDoOnError.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose.subscribeActual(SingleDoOnDispose.java:38) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleMap.subscribeActual(SingleMap.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleCreate$Emitter.onSuccess(SingleCreate.java:68) at com.google.devtools.build.lib.remote.grpc.TokenBucket$1.onNext(TokenBucket.java:79) at io.reactivex.rxjava3.internal.util.NotificationLite.accept(NotificationLite.java:247) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.test(BehaviorSubject.java:507) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.emitFirst(BehaviorSubject.java:468) at io.reactivex.rxjava3.subjects.BehaviorSubject.subscribeActual(BehaviorSubject.java:224) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at com.google.devtools.build.lib.remote.grpc.TokenBucket.lambda$acquireToken$0(TokenBucket.java:64) at io.reactivex.rxjava3.internal.operators.single.SingleCreate.subscribeActual(SingleCreate.java:40) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDefer.subscribeActual(SingleDefer.java:43) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.core.Single.blockingGet(Single.java:3644) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.withChannelBlocking(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$1(RemoteServerCapabilities.java:83) at com.google.devtools.build.lib.remote.Retrier.execute(Retrier.java:244) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:125) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:114) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.get(RemoteServerCapabilities.java:81) at com.google.devtools.build.lib.remote.RemoteModule.getAndVerifyServerCapabilities(RemoteModule.java:209) at com.google.devtools.build.lib.remote.RemoteModule.beforeCommand(RemoteModule.java:542) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.execExclusively(BlazeCommandDispatcher.java:379) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.exec(BlazeCommandDispatcher.java:233) at com.google.devtools.build.lib.server.GrpcServerImpl.executeCommand(GrpcServerImpl.java:550) at com.google.devtools.build.lib.server.GrpcServerImpl.lambda$run$1(GrpcServerImpl.java:614) at io.grpc.Context$1.run(Context.java:566) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) ERROR: Failed to query remote execution capabilities: Stream closed java.io.IOException: Stream closed at java.base/java.lang.ProcessBuilder$NullOutputStream.write(Unknown Source) at java.base/java.io.OutputStream.write(Unknown Source) at java.base/java.io.BufferedOutputStream.flushBuffer(Unknown Source) at java.base/java.io.BufferedOutputStream.flush(Unknown Source) at java.base/java.io.FilterOutputStream.close(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.implClose(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.close(Unknown Source) at java.base/java.io.OutputStreamWriter.close(Unknown Source) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelper.getCredentials(CredentialHelper.java:82) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getCredentialsFromHelper(CredentialHelperCredentials.java:108) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2413) at java.base/java.util.concurrent.ConcurrentHashMap.compute(Unknown Source) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2411) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2394) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getRequestMetadata(CredentialHelperCredentials.java:80) at com.google.auth.Credentials.blockingGetToCallback(Credentials.java:112) at com.google.auth.Credentials$1.run(Credentials.java:98) at io.grpc.stub.ClientCalls$ThreadlessExecutor.runQuietly(ClientCalls.java:771) at io.grpc.stub.ClientCalls$ThreadlessExecutor.waitAndDrain(ClientCalls.java:754) at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:157) at build.bazel.remote.execution.v2.CapabilitiesGrpc$CapabilitiesBlockingStub.getCapabilities(CapabilitiesGrpc.java:218) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$0(RemoteServerCapabilities.java:84) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannelBlocking$2(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannel$4(ReferenceCountedChannel.java:108) at io.reactivex.rxjava3.internal.operators.single.SingleUsing.subscribeActual(SingleUsing.java:59) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:112) at io.reactivex.rxjava3.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:65) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose$DoOnDisposeObserver.onSuccess(SingleDoOnDispose.java:84) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError$DoOnError.onSuccess(SingleDoOnError.java:52) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle$SingleElementObserver.onComplete(ObservableSingleSingle.java:110) at io.reactivex.rxjava3.internal.observers.DeferredScalarDisposable.complete(DeferredScalarDisposable.java:85) at io.reactivex.rxjava3.subjects.AsyncSubject.subscribeActual(AsyncSubject.java:233) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle.subscribeActual(ObservableSingleSingle.java:36) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError.subscribeActual(SingleDoOnError.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose.subscribeActual(SingleDoOnDispose.java:38) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleMap.subscribeActual(SingleMap.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleCreate$Emitter.onSuccess(SingleCreate.java:68) at com.google.devtools.build.lib.remote.grpc.TokenBucket$1.onNext(TokenBucket.java:79) at io.reactivex.rxjava3.internal.util.NotificationLite.accept(NotificationLite.java:247) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.test(BehaviorSubject.java:507) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.emitFirst(BehaviorSubject.java:468) at io.reactivex.rxjava3.subjects.BehaviorSubject.subscribeActual(BehaviorSubject.java:224) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at com.google.devtools.build.lib.remote.grpc.TokenBucket.lambda$acquireToken$0(TokenBucket.java:64) at io.reactivex.rxjava3.internal.operators.single.SingleCreate.subscribeActual(SingleCreate.java:40) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDefer.subscribeActual(SingleDefer.java:43) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.core.Single.blockingGet(Single.java:3644) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.withChannelBlocking(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$1(RemoteServerCapabilities.java:83) at com.google.devtools.build.lib.remote.Retrier.execute(Retrier.java:244) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:125) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:114) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.get(RemoteServerCapabilities.java:81) at com.google.devtools.build.lib.remote.RemoteModule.getAndVerifyServerCapabilities(RemoteModule.java:209) at com.google.devtools.build.lib.remote.RemoteModule.beforeCommand(RemoteModule.java:542) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.execExclusively(BlazeCommandDispatcher.java:379) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.exec(BlazeCommandDispatcher.java:233) at com.google.devtools.build.lib.server.GrpcServerImpl.executeCommand(GrpcServerImpl.java:550) at com.google.devtools.build.lib.server.GrpcServerImpl.lambda$run$1(GrpcServerImpl.java:614) at io.grpc.Context$1.run(Context.java:566) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) INFO: Streaming build results to: https://opal.cluster.engflow.com/invocation/86e227bc-187e-4d28-8f04-40b1ce9a36a4 ```
|
@tjgq PTAL Also: if you have any idea how to create a unit test for this race condition I'm very happy to add one. I saw this in the wild flakily, and basically only when starting a new Bazel server. |
sgowroji
added
team-Remote-Exec
I'm assuming the existing test didn't catch it because a Python process isn't fast enough to win the race? The best idea I have is to rewrite the test helper in C++, but that could be annoying to set up. (I'm okay with merging as-is.) |
|
I'm not convinced the language matters too much. I was able to repro this with a shell script |
|
@bazel-io flag |
bazel-io
added
the
potential release blocker
|
(we should cherry-pick to 6.x and also 5.x in case there will be another 5.x release) |
tjgq
approved these changes
Mar 6, 2023
|
@bazel-io fork 6.2.0 |
bazel-io
removed
the
potential release blocker
keertk
added
potential release blocker
ShreeM01
pushed a commit
to ShreeM01/bazel
that referenced
this pull request
Mar 11, 2023
This fixes a race condition where writing to stdin of the subprocess fails because the subprocess terminated quickly (e.g., because it did not need the request to provide credentials, or some other error occurred in the helper before it reads from stdin). ``` Starting local Bazel server and connecting to it... INFO: Writing tracer profile to '/private/var/tmp/_bazel_yannic/bb6c3b02ccc98aaca966df445b5024eb/command.profile.gz' INFO: Invocation ID: 86e227bc-187e-4d28-8f04-40b1ce9a36a4 ERROR: java.io.IOException: Stream closed at java.base/java.lang.ProcessBuilder$NullOutputStream.write(Unknown Source) at java.base/java.io.OutputStream.write(Unknown Source) at java.base/java.io.BufferedOutputStream.flushBuffer(Unknown Source) at java.base/java.io.BufferedOutputStream.flush(Unknown Source) at java.base/java.io.FilterOutputStream.close(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.implClose(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.close(Unknown Source) at java.base/java.io.OutputStreamWriter.close(Unknown Source) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelper.getCredentials(CredentialHelper.java:82) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getCredentialsFromHelper(CredentialHelperCredentials.java:108) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2413) at java.base/java.util.concurrent.ConcurrentHashMap.compute(Unknown Source) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2411) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2394) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getRequestMetadata(CredentialHelperCredentials.java:80) at com.google.auth.Credentials.blockingGetToCallback(Credentials.java:112) at com.google.auth.Credentials$1.run(Credentials.java:98) at io.grpc.stub.ClientCalls$ThreadlessExecutor.runQuietly(ClientCalls.java:771) at io.grpc.stub.ClientCalls$ThreadlessExecutor.waitAndDrain(ClientCalls.java:754) at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:157) at build.bazel.remote.execution.v2.CapabilitiesGrpc$CapabilitiesBlockingStub.getCapabilities(CapabilitiesGrpc.java:218) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$0(RemoteServerCapabilities.java:84) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannelBlocking$2(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannel$4(ReferenceCountedChannel.java:108) at io.reactivex.rxjava3.internal.operators.single.SingleUsing.subscribeActual(SingleUsing.java:59) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:112) at io.reactivex.rxjava3.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:65) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose$DoOnDisposeObserver.onSuccess(SingleDoOnDispose.java:84) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError$DoOnError.onSuccess(SingleDoOnError.java:52) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle$SingleElementObserver.onComplete(ObservableSingleSingle.java:110) at io.reactivex.rxjava3.internal.observers.DeferredScalarDisposable.complete(DeferredScalarDisposable.java:85) at io.reactivex.rxjava3.subjects.AsyncSubject.subscribeActual(AsyncSubject.java:233) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle.subscribeActual(ObservableSingleSingle.java:36) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError.subscribeActual(SingleDoOnError.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose.subscribeActual(SingleDoOnDispose.java:38) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleMap.subscribeActual(SingleMap.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleCreate$Emitter.onSuccess(SingleCreate.java:68) at com.google.devtools.build.lib.remote.grpc.TokenBucket$1.onNext(TokenBucket.java:79) at io.reactivex.rxjava3.internal.util.NotificationLite.accept(NotificationLite.java:247) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.test(BehaviorSubject.java:507) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.emitFirst(BehaviorSubject.java:468) at io.reactivex.rxjava3.subjects.BehaviorSubject.subscribeActual(BehaviorSubject.java:224) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at com.google.devtools.build.lib.remote.grpc.TokenBucket.lambda$acquireToken$0(TokenBucket.java:64) at io.reactivex.rxjava3.internal.operators.single.SingleCreate.subscribeActual(SingleCreate.java:40) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDefer.subscribeActual(SingleDefer.java:43) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.core.Single.blockingGet(Single.java:3644) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.withChannelBlocking(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$1(RemoteServerCapabilities.java:83) at com.google.devtools.build.lib.remote.Retrier.execute(Retrier.java:244) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:125) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:114) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.get(RemoteServerCapabilities.java:81) at com.google.devtools.build.lib.remote.RemoteModule.getAndVerifyServerCapabilities(RemoteModule.java:209) at com.google.devtools.build.lib.remote.RemoteModule.beforeCommand(RemoteModule.java:542) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.execExclusively(BlazeCommandDispatcher.java:379) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.exec(BlazeCommandDispatcher.java:233) at com.google.devtools.build.lib.server.GrpcServerImpl.executeCommand(GrpcServerImpl.java:550) at com.google.devtools.build.lib.server.GrpcServerImpl.lambda$run$1(GrpcServerImpl.java:614) at io.grpc.Context$1.run(Context.java:566) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) ERROR: Failed to query remote execution capabilities: Stream closed java.io.IOException: Stream closed at java.base/java.lang.ProcessBuilder$NullOutputStream.write(Unknown Source) at java.base/java.io.OutputStream.write(Unknown Source) at java.base/java.io.BufferedOutputStream.flushBuffer(Unknown Source) at java.base/java.io.BufferedOutputStream.flush(Unknown Source) at java.base/java.io.FilterOutputStream.close(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.implClose(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.close(Unknown Source) at java.base/java.io.OutputStreamWriter.close(Unknown Source) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelper.getCredentials(CredentialHelper.java:82) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getCredentialsFromHelper(CredentialHelperCredentials.java:108) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2413) at java.base/java.util.concurrent.ConcurrentHashMap.compute(Unknown Source) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2411) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2394) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getRequestMetadata(CredentialHelperCredentials.java:80) at com.google.auth.Credentials.blockingGetToCallback(Credentials.java:112) at com.google.auth.Credentials$1.run(Credentials.java:98) at io.grpc.stub.ClientCalls$ThreadlessExecutor.runQuietly(ClientCalls.java:771) at io.grpc.stub.ClientCalls$ThreadlessExecutor.waitAndDrain(ClientCalls.java:754) at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:157) at build.bazel.remote.execution.v2.CapabilitiesGrpc$CapabilitiesBlockingStub.getCapabilities(CapabilitiesGrpc.java:218) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$0(RemoteServerCapabilities.java:84) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannelBlocking$2(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannel$4(ReferenceCountedChannel.java:108) at io.reactivex.rxjava3.internal.operators.single.SingleUsing.subscribeActual(SingleUsing.java:59) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:112) at io.reactivex.rxjava3.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:65) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose$DoOnDisposeObserver.onSuccess(SingleDoOnDispose.java:84) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError$DoOnError.onSuccess(SingleDoOnError.java:52) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle$SingleElementObserver.onComplete(ObservableSingleSingle.java:110) at io.reactivex.rxjava3.internal.observers.DeferredScalarDisposable.complete(DeferredScalarDisposable.java:85) at io.reactivex.rxjava3.subjects.AsyncSubject.subscribeActual(AsyncSubject.java:233) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle.subscribeActual(ObservableSingleSingle.java:36) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError.subscribeActual(SingleDoOnError.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose.subscribeActual(SingleDoOnDispose.java:38) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleMap.subscribeActual(SingleMap.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleCreate$Emitter.onSuccess(SingleCreate.java:68) at com.google.devtools.build.lib.remote.grpc.TokenBucket$1.onNext(TokenBucket.java:79) at io.reactivex.rxjava3.internal.util.NotificationLite.accept(NotificationLite.java:247) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.test(BehaviorSubject.java:507) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.emitFirst(BehaviorSubject.java:468) at io.reactivex.rxjava3.subjects.BehaviorSubject.subscribeActual(BehaviorSubject.java:224) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at com.google.devtools.build.lib.remote.grpc.TokenBucket.lambda$acquireToken$0(TokenBucket.java:64) at io.reactivex.rxjava3.internal.operators.single.SingleCreate.subscribeActual(SingleCreate.java:40) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDefer.subscribeActual(SingleDefer.java:43) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.core.Single.blockingGet(Single.java:3644) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.withChannelBlocking(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$1(RemoteServerCapabilities.java:83) at com.google.devtools.build.lib.remote.Retrier.execute(Retrier.java:244) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:125) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:114) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.get(RemoteServerCapabilities.java:81) at com.google.devtools.build.lib.remote.RemoteModule.getAndVerifyServerCapabilities(RemoteModule.java:209) at com.google.devtools.build.lib.remote.RemoteModule.beforeCommand(RemoteModule.java:542) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.execExclusively(BlazeCommandDispatcher.java:379) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.exec(BlazeCommandDispatcher.java:233) at com.google.devtools.build.lib.server.GrpcServerImpl.executeCommand(GrpcServerImpl.java:550) at com.google.devtools.build.lib.server.GrpcServerImpl.lambda$run$1(GrpcServerImpl.java:614) at io.grpc.Context$1.run(Context.java:566) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) INFO: Streaming build results to: https://opal.cluster.engflow.com/invocation/86e227bc-187e-4d28-8f04-40b1ce9a36a4 ``` Closes bazelbuild#17666. PiperOrigin-RevId: 514377908 Change-Id: I772a5e462146e3e9699b45090120634a057a5532
ShreeM01
added a commit
that referenced
this pull request
Mar 13, 2023
This fixes a race condition where writing to stdin of the subprocess fails because the subprocess terminated quickly (e.g., because it did not need the request to provide credentials, or some other error occurred in the helper before it reads from stdin). ``` Starting local Bazel server and connecting to it... INFO: Writing tracer profile to '/private/var/tmp/_bazel_yannic/bb6c3b02ccc98aaca966df445b5024eb/command.profile.gz' INFO: Invocation ID: 86e227bc-187e-4d28-8f04-40b1ce9a36a4 ERROR: java.io.IOException: Stream closed at java.base/java.lang.ProcessBuilder$NullOutputStream.write(Unknown Source) at java.base/java.io.OutputStream.write(Unknown Source) at java.base/java.io.BufferedOutputStream.flushBuffer(Unknown Source) at java.base/java.io.BufferedOutputStream.flush(Unknown Source) at java.base/java.io.FilterOutputStream.close(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.implClose(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.close(Unknown Source) at java.base/java.io.OutputStreamWriter.close(Unknown Source) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelper.getCredentials(CredentialHelper.java:82) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getCredentialsFromHelper(CredentialHelperCredentials.java:108) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2413) at java.base/java.util.concurrent.ConcurrentHashMap.compute(Unknown Source) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2411) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2394) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getRequestMetadata(CredentialHelperCredentials.java:80) at com.google.auth.Credentials.blockingGetToCallback(Credentials.java:112) at com.google.auth.Credentials$1.run(Credentials.java:98) at io.grpc.stub.ClientCalls$ThreadlessExecutor.runQuietly(ClientCalls.java:771) at io.grpc.stub.ClientCalls$ThreadlessExecutor.waitAndDrain(ClientCalls.java:754) at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:157) at build.bazel.remote.execution.v2.CapabilitiesGrpc$CapabilitiesBlockingStub.getCapabilities(CapabilitiesGrpc.java:218) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$0(RemoteServerCapabilities.java:84) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannelBlocking$2(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannel$4(ReferenceCountedChannel.java:108) at io.reactivex.rxjava3.internal.operators.single.SingleUsing.subscribeActual(SingleUsing.java:59) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:112) at io.reactivex.rxjava3.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:65) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose$DoOnDisposeObserver.onSuccess(SingleDoOnDispose.java:84) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError$DoOnError.onSuccess(SingleDoOnError.java:52) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle$SingleElementObserver.onComplete(ObservableSingleSingle.java:110) at io.reactivex.rxjava3.internal.observers.DeferredScalarDisposable.complete(DeferredScalarDisposable.java:85) at io.reactivex.rxjava3.subjects.AsyncSubject.subscribeActual(AsyncSubject.java:233) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle.subscribeActual(ObservableSingleSingle.java:36) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError.subscribeActual(SingleDoOnError.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose.subscribeActual(SingleDoOnDispose.java:38) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleMap.subscribeActual(SingleMap.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleCreate$Emitter.onSuccess(SingleCreate.java:68) at com.google.devtools.build.lib.remote.grpc.TokenBucket$1.onNext(TokenBucket.java:79) at io.reactivex.rxjava3.internal.util.NotificationLite.accept(NotificationLite.java:247) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.test(BehaviorSubject.java:507) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.emitFirst(BehaviorSubject.java:468) at io.reactivex.rxjava3.subjects.BehaviorSubject.subscribeActual(BehaviorSubject.java:224) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at com.google.devtools.build.lib.remote.grpc.TokenBucket.lambda$acquireToken$0(TokenBucket.java:64) at io.reactivex.rxjava3.internal.operators.single.SingleCreate.subscribeActual(SingleCreate.java:40) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDefer.subscribeActual(SingleDefer.java:43) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.core.Single.blockingGet(Single.java:3644) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.withChannelBlocking(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$1(RemoteServerCapabilities.java:83) at com.google.devtools.build.lib.remote.Retrier.execute(Retrier.java:244) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:125) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:114) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.get(RemoteServerCapabilities.java:81) at com.google.devtools.build.lib.remote.RemoteModule.getAndVerifyServerCapabilities(RemoteModule.java:209) at com.google.devtools.build.lib.remote.RemoteModule.beforeCommand(RemoteModule.java:542) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.execExclusively(BlazeCommandDispatcher.java:379) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.exec(BlazeCommandDispatcher.java:233) at com.google.devtools.build.lib.server.GrpcServerImpl.executeCommand(GrpcServerImpl.java:550) at com.google.devtools.build.lib.server.GrpcServerImpl.lambda$run$1(GrpcServerImpl.java:614) at io.grpc.Context$1.run(Context.java:566) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) ERROR: Failed to query remote execution capabilities: Stream closed java.io.IOException: Stream closed at java.base/java.lang.ProcessBuilder$NullOutputStream.write(Unknown Source) at java.base/java.io.OutputStream.write(Unknown Source) at java.base/java.io.BufferedOutputStream.flushBuffer(Unknown Source) at java.base/java.io.BufferedOutputStream.flush(Unknown Source) at java.base/java.io.FilterOutputStream.close(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.implClose(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.close(Unknown Source) at java.base/java.io.OutputStreamWriter.close(Unknown Source) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelper.getCredentials(CredentialHelper.java:82) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getCredentialsFromHelper(CredentialHelperCredentials.java:108) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2413) at java.base/java.util.concurrent.ConcurrentHashMap.compute(Unknown Source) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2411) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2394) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getRequestMetadata(CredentialHelperCredentials.java:80) at com.google.auth.Credentials.blockingGetToCallback(Credentials.java:112) at com.google.auth.Credentials$1.run(Credentials.java:98) at io.grpc.stub.ClientCalls$ThreadlessExecutor.runQuietly(ClientCalls.java:771) at io.grpc.stub.ClientCalls$ThreadlessExecutor.waitAndDrain(ClientCalls.java:754) at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:157) at build.bazel.remote.execution.v2.CapabilitiesGrpc$CapabilitiesBlockingStub.getCapabilities(CapabilitiesGrpc.java:218) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$0(RemoteServerCapabilities.java:84) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannelBlocking$2(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannel$4(ReferenceCountedChannel.java:108) at io.reactivex.rxjava3.internal.operators.single.SingleUsing.subscribeActual(SingleUsing.java:59) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:112) at io.reactivex.rxjava3.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:65) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose$DoOnDisposeObserver.onSuccess(SingleDoOnDispose.java:84) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError$DoOnError.onSuccess(SingleDoOnError.java:52) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle$SingleElementObserver.onComplete(ObservableSingleSingle.java:110) at io.reactivex.rxjava3.internal.observers.DeferredScalarDisposable.complete(DeferredScalarDisposable.java:85) at io.reactivex.rxjava3.subjects.AsyncSubject.subscribeActual(AsyncSubject.java:233) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle.subscribeActual(ObservableSingleSingle.java:36) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError.subscribeActual(SingleDoOnError.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose.subscribeActual(SingleDoOnDispose.java:38) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleMap.subscribeActual(SingleMap.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleCreate$Emitter.onSuccess(SingleCreate.java:68) at com.google.devtools.build.lib.remote.grpc.TokenBucket$1.onNext(TokenBucket.java:79) at io.reactivex.rxjava3.internal.util.NotificationLite.accept(NotificationLite.java:247) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.test(BehaviorSubject.java:507) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.emitFirst(BehaviorSubject.java:468) at io.reactivex.rxjava3.subjects.BehaviorSubject.subscribeActual(BehaviorSubject.java:224) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at com.google.devtools.build.lib.remote.grpc.TokenBucket.lambda$acquireToken$0(TokenBucket.java:64) at io.reactivex.rxjava3.internal.operators.single.SingleCreate.subscribeActual(SingleCreate.java:40) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDefer.subscribeActual(SingleDefer.java:43) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.core.Single.blockingGet(Single.java:3644) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.withChannelBlocking(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$1(RemoteServerCapabilities.java:83) at com.google.devtools.build.lib.remote.Retrier.execute(Retrier.java:244) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:125) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:114) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.get(RemoteServerCapabilities.java:81) at com.google.devtools.build.lib.remote.RemoteModule.getAndVerifyServerCapabilities(RemoteModule.java:209) at com.google.devtools.build.lib.remote.RemoteModule.beforeCommand(RemoteModule.java:542) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.execExclusively(BlazeCommandDispatcher.java:379) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.exec(BlazeCommandDispatcher.java:233) at com.google.devtools.build.lib.server.GrpcServerImpl.executeCommand(GrpcServerImpl.java:550) at com.google.devtools.build.lib.server.GrpcServerImpl.lambda$run$1(GrpcServerImpl.java:614) at io.grpc.Context$1.run(Context.java:566) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) INFO: Streaming build results to: https://opal.cluster.engflow.com/invocation/86e227bc-187e-4d28-8f04-40b1ce9a36a4 ``` Closes #17666. PiperOrigin-RevId: 514377908 Change-Id: I772a5e462146e3e9699b45090120634a057a5532 Co-authored-by: Yannic Bonenberger <yannic@engflow.com>
fweikert
pushed a commit
to fweikert/bazel
that referenced
this pull request
May 25, 2023
This fixes a race condition where writing to stdin of the subprocess fails because the subprocess terminated quickly (e.g., because it did not need the request to provide credentials, or some other error occurred in the helper before it reads from stdin). ``` Starting local Bazel server and connecting to it... INFO: Writing tracer profile to '/private/var/tmp/_bazel_yannic/bb6c3b02ccc98aaca966df445b5024eb/command.profile.gz' INFO: Invocation ID: 86e227bc-187e-4d28-8f04-40b1ce9a36a4 ERROR: java.io.IOException: Stream closed at java.base/java.lang.ProcessBuilder$NullOutputStream.write(Unknown Source) at java.base/java.io.OutputStream.write(Unknown Source) at java.base/java.io.BufferedOutputStream.flushBuffer(Unknown Source) at java.base/java.io.BufferedOutputStream.flush(Unknown Source) at java.base/java.io.FilterOutputStream.close(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.implClose(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.close(Unknown Source) at java.base/java.io.OutputStreamWriter.close(Unknown Source) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelper.getCredentials(CredentialHelper.java:82) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getCredentialsFromHelper(CredentialHelperCredentials.java:108) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2413) at java.base/java.util.concurrent.ConcurrentHashMap.compute(Unknown Source) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2411) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2394) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getRequestMetadata(CredentialHelperCredentials.java:80) at com.google.auth.Credentials.blockingGetToCallback(Credentials.java:112) at com.google.auth.Credentials$1.run(Credentials.java:98) at io.grpc.stub.ClientCalls$ThreadlessExecutor.runQuietly(ClientCalls.java:771) at io.grpc.stub.ClientCalls$ThreadlessExecutor.waitAndDrain(ClientCalls.java:754) at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:157) at build.bazel.remote.execution.v2.CapabilitiesGrpc$CapabilitiesBlockingStub.getCapabilities(CapabilitiesGrpc.java:218) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$0(RemoteServerCapabilities.java:84) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannelBlocking$2(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannel$4(ReferenceCountedChannel.java:108) at io.reactivex.rxjava3.internal.operators.single.SingleUsing.subscribeActual(SingleUsing.java:59) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:112) at io.reactivex.rxjava3.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:65) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose$DoOnDisposeObserver.onSuccess(SingleDoOnDispose.java:84) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError$DoOnError.onSuccess(SingleDoOnError.java:52) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle$SingleElementObserver.onComplete(ObservableSingleSingle.java:110) at io.reactivex.rxjava3.internal.observers.DeferredScalarDisposable.complete(DeferredScalarDisposable.java:85) at io.reactivex.rxjava3.subjects.AsyncSubject.subscribeActual(AsyncSubject.java:233) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle.subscribeActual(ObservableSingleSingle.java:36) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError.subscribeActual(SingleDoOnError.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose.subscribeActual(SingleDoOnDispose.java:38) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleMap.subscribeActual(SingleMap.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleCreate$Emitter.onSuccess(SingleCreate.java:68) at com.google.devtools.build.lib.remote.grpc.TokenBucket$1.onNext(TokenBucket.java:79) at io.reactivex.rxjava3.internal.util.NotificationLite.accept(NotificationLite.java:247) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.test(BehaviorSubject.java:507) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.emitFirst(BehaviorSubject.java:468) at io.reactivex.rxjava3.subjects.BehaviorSubject.subscribeActual(BehaviorSubject.java:224) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at com.google.devtools.build.lib.remote.grpc.TokenBucket.lambda$acquireToken$0(TokenBucket.java:64) at io.reactivex.rxjava3.internal.operators.single.SingleCreate.subscribeActual(SingleCreate.java:40) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDefer.subscribeActual(SingleDefer.java:43) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.core.Single.blockingGet(Single.java:3644) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.withChannelBlocking(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$1(RemoteServerCapabilities.java:83) at com.google.devtools.build.lib.remote.Retrier.execute(Retrier.java:244) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:125) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:114) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.get(RemoteServerCapabilities.java:81) at com.google.devtools.build.lib.remote.RemoteModule.getAndVerifyServerCapabilities(RemoteModule.java:209) at com.google.devtools.build.lib.remote.RemoteModule.beforeCommand(RemoteModule.java:542) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.execExclusively(BlazeCommandDispatcher.java:379) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.exec(BlazeCommandDispatcher.java:233) at com.google.devtools.build.lib.server.GrpcServerImpl.executeCommand(GrpcServerImpl.java:550) at com.google.devtools.build.lib.server.GrpcServerImpl.lambda$run$1(GrpcServerImpl.java:614) at io.grpc.Context$1.run(Context.java:566) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) ERROR: Failed to query remote execution capabilities: Stream closed java.io.IOException: Stream closed at java.base/java.lang.ProcessBuilder$NullOutputStream.write(Unknown Source) at java.base/java.io.OutputStream.write(Unknown Source) at java.base/java.io.BufferedOutputStream.flushBuffer(Unknown Source) at java.base/java.io.BufferedOutputStream.flush(Unknown Source) at java.base/java.io.FilterOutputStream.close(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.implClose(Unknown Source) at java.base/sun.nio.cs.StreamEncoder.close(Unknown Source) at java.base/java.io.OutputStreamWriter.close(Unknown Source) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelper.getCredentials(CredentialHelper.java:82) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getCredentialsFromHelper(CredentialHelperCredentials.java:108) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2413) at java.base/java.util.concurrent.ConcurrentHashMap.compute(Unknown Source) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2411) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2394) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62) at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getRequestMetadata(CredentialHelperCredentials.java:80) at com.google.auth.Credentials.blockingGetToCallback(Credentials.java:112) at com.google.auth.Credentials$1.run(Credentials.java:98) at io.grpc.stub.ClientCalls$ThreadlessExecutor.runQuietly(ClientCalls.java:771) at io.grpc.stub.ClientCalls$ThreadlessExecutor.waitAndDrain(ClientCalls.java:754) at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:157) at build.bazel.remote.execution.v2.CapabilitiesGrpc$CapabilitiesBlockingStub.getCapabilities(CapabilitiesGrpc.java:218) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$0(RemoteServerCapabilities.java:84) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannelBlocking$2(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannel$4(ReferenceCountedChannel.java:108) at io.reactivex.rxjava3.internal.operators.single.SingleUsing.subscribeActual(SingleUsing.java:59) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:112) at io.reactivex.rxjava3.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:65) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose$DoOnDisposeObserver.onSuccess(SingleDoOnDispose.java:84) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError$DoOnError.onSuccess(SingleDoOnError.java:52) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle$SingleElementObserver.onComplete(ObservableSingleSingle.java:110) at io.reactivex.rxjava3.internal.observers.DeferredScalarDisposable.complete(DeferredScalarDisposable.java:85) at io.reactivex.rxjava3.subjects.AsyncSubject.subscribeActual(AsyncSubject.java:233) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle.subscribeActual(ObservableSingleSingle.java:36) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError.subscribeActual(SingleDoOnError.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose.subscribeActual(SingleDoOnDispose.java:38) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleMap.subscribeActual(SingleMap.java:35) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85) at io.reactivex.rxjava3.internal.operators.single.SingleCreate$Emitter.onSuccess(SingleCreate.java:68) at com.google.devtools.build.lib.remote.grpc.TokenBucket$1.onNext(TokenBucket.java:79) at io.reactivex.rxjava3.internal.util.NotificationLite.accept(NotificationLite.java:247) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.test(BehaviorSubject.java:507) at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.emitFirst(BehaviorSubject.java:468) at io.reactivex.rxjava3.subjects.BehaviorSubject.subscribeActual(BehaviorSubject.java:224) at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176) at com.google.devtools.build.lib.remote.grpc.TokenBucket.lambda$acquireToken$0(TokenBucket.java:64) at io.reactivex.rxjava3.internal.operators.single.SingleCreate.subscribeActual(SingleCreate.java:40) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleDefer.subscribeActual(SingleDefer.java:43) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37) at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855) at io.reactivex.rxjava3.core.Single.blockingGet(Single.java:3644) at com.google.devtools.build.lib.remote.ReferenceCountedChannel.withChannelBlocking(ReferenceCountedChannel.java:85) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$1(RemoteServerCapabilities.java:83) at com.google.devtools.build.lib.remote.Retrier.execute(Retrier.java:244) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:125) at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:114) at com.google.devtools.build.lib.remote.RemoteServerCapabilities.get(RemoteServerCapabilities.java:81) at com.google.devtools.build.lib.remote.RemoteModule.getAndVerifyServerCapabilities(RemoteModule.java:209) at com.google.devtools.build.lib.remote.RemoteModule.beforeCommand(RemoteModule.java:542) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.execExclusively(BlazeCommandDispatcher.java:379) at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.exec(BlazeCommandDispatcher.java:233) at com.google.devtools.build.lib.server.GrpcServerImpl.executeCommand(GrpcServerImpl.java:550) at com.google.devtools.build.lib.server.GrpcServerImpl.lambda$run$1(GrpcServerImpl.java:614) at io.grpc.Context$1.run(Context.java:566) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) INFO: Streaming build results to: https://opal.cluster.engflow.com/invocation/86e227bc-187e-4d28-8f04-40b1ce9a36a4 ``` Closes bazelbuild#17666. PiperOrigin-RevId: 514377908 Change-Id: I772a5e462146e3e9699b45090120634a057a5532
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
This fixes a race condition where writing to stdin of the subprocess fails because the subprocess terminated quickly (e.g., because it did not need the request to provide credentials, or some other error occurred in the helper before it reads from stdin).