[Ignore] Repro HttpStress ODE

eng/pipelines/libraries/stress/http.yml

@@ -30,73 +30,73 @@ extends:
     stages:
     - stage: Build
       jobs:
-      - job: linux
-        displayName: Docker Linux
-        timeoutInMinutes: 240
-        variables:
-          DUMPS_SHARE_MOUNT_ROOT: "/dumps-share"
-          DUMPS_SHARE: "$(Build.ArtifactStagingDirectory)/dumps/"
-        pool:
-          name: $(DncEngPublicBuildPool)
-          demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open
-
-        steps:
-        - checkout: self
-          clean: true
-          fetchDepth: 5
-
-        - bash: |
-            $(dockerfilesFolder)/build-docker-sdk.sh -t $(sdkBaseImage) -c $(BUILD_CONFIGURATION) && \
-            echo "##vso[task.setvariable variable=succeeded;isOutput=true]true"
-          name: buildRuntime
-          displayName: Build CLR and Libraries
-
-        - bash: |
-            $(httpStressProject)/run-docker-compose.sh -o -c $(BUILD_CONFIGURATION) -t $(sdkBaseImage) && \
-            echo "##vso[task.setvariable variable=succeeded;isOutput=true]true"
-          name: buildStress
-          displayName: Build HttpStress
-
-        - bash: |
-            cd '$(httpStressProject)'
-            export STRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
-            export STRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0"
-            mkdir -p $DUMPS_SHARE
-            docker-compose up --abort-on-container-exit --no-color
-          timeoutInMinutes: 35 # In case the HTTP/3.0 run hangs, we timeout shortly after the expected 30 minute run
-          displayName: Run HttpStress - HTTP 3.0
-          condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
-
-        - bash: |
-            cd '$(httpStressProject)'
-            export STRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 2.0"
-            export STRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 2.0"
-            mkdir -p $DUMPS_SHARE
-            docker-compose up --abort-on-container-exit --no-color
-          displayName: Run HttpStress - HTTP 2.0
-          condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
-
-        - bash: |
-            cd '$(httpStressProject)'
-            export STRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 1.1"
-            export STRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 1.1"
-            mkdir -p $DUMPS_SHARE
-            docker-compose up --abort-on-container-exit --no-color
-          displayName: Run HttpStress - HTTP 1.1
-          condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
-
-        - bash: |
-            sudo chown -R $(id -u):$(id -g) $(Build.ArtifactStagingDirectory)/dumps
-          displayName: Own shared dumps and files
-          condition: failed()
-
-        - publish: $(Build.ArtifactStagingDirectory)/dumps
-          artifact: DumpsLinux
-          condition: failed()
+      # - job: linux
+      #   displayName: Docker Linux
+      #   timeoutInMinutes: 240
+      #   variables:
+      #     DUMPS_SHARE_MOUNT_ROOT: "/dumps-share"
+      #     DUMPS_SHARE: "$(Build.ArtifactStagingDirectory)/dumps/"
+      #   pool:
+      #     name: $(DncEngPublicBuildPool)
+      #     demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open
+
+      #   steps:
+      #   - checkout: self
+      #     clean: true
+      #     fetchDepth: 5
+
+      #   - bash: |
+      #       $(dockerfilesFolder)/build-docker-sdk.sh -t $(sdkBaseImage) -c $(BUILD_CONFIGURATION) && \
+      #       echo "##vso[task.setvariable variable=succeeded;isOutput=true]true"
+      #     name: buildRuntime
+      #     displayName: Build CLR and Libraries
+
+      #   - bash: |
+      #       $(httpStressProject)/run-docker-compose.sh -o -c $(BUILD_CONFIGURATION) -t $(sdkBaseImage) && \
+      #       echo "##vso[task.setvariable variable=succeeded;isOutput=true]true"
+      #     name: buildStress
+      #     displayName: Build HttpStress
+
+      #   - bash: |
+      #       cd '$(httpStressProject)'
+      #       export STRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
+      #       export STRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0"
+      #       mkdir -p $DUMPS_SHARE
+      #       docker-compose up --abort-on-container-exit --no-color
+      #     timeoutInMinutes: 35 # In case the HTTP/3.0 run hangs, we timeout shortly after the expected 30 minute run
+      #     displayName: Run HttpStress - HTTP 3.0
+      #     condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
+
+      #   - bash: |
+      #       cd '$(httpStressProject)'
+      #       export STRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 2.0"
+      #       export STRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 2.0"
+      #       mkdir -p $DUMPS_SHARE
+      #       docker-compose up --abort-on-container-exit --no-color
+      #     displayName: Run HttpStress - HTTP 2.0
+      #     condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
+
+      #   - bash: |
+      #       cd '$(httpStressProject)'
+      #       export STRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 1.1"
+      #       export STRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 1.1"
+      #       mkdir -p $DUMPS_SHARE
+      #       docker-compose up --abort-on-container-exit --no-color
+      #     displayName: Run HttpStress - HTTP 1.1
+      #     condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
+
+      #   - bash: |
+      #       sudo chown -R $(id -u):$(id -g) $(Build.ArtifactStagingDirectory)/dumps
+      #     displayName: Own shared dumps and files
+      #     condition: failed()
+
+      #   - publish: $(Build.ArtifactStagingDirectory)/dumps
+      #     artifact: DumpsLinux
+      #     condition: failed()
 
       - job: windows
         displayName: Docker NanoServer
-        timeoutInMinutes: 150
+        timeoutInMinutes: 250
         variables:
           DUMPS_SHARE_MOUNT_ROOT: "C:/dumps-share"
           DUMPS_SHARE: "$(Build.ArtifactStagingDirectory)/dumps/"
@@ -129,27 +129,27 @@ extends:
           name: disableFirewall
           displayName: Disable Firewall
 
-        - powershell: |
-            cd '$(httpStressProject)'
-            $env:STRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 3.0"
-            $env:STRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 3.0"
-            New-Item -Force $env:DUMPS_SHARE -ItemType Directory
-            docker compose up --abort-on-container-exit --no-color
-          displayName: Run HttpStress - HTTP 3.0
-          condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
-
-        - powershell: |
-            cd '$(httpStressProject)'
-            $env:STRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 2.0"
-            $env:STRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 2.0"
-            New-Item -Force $env:DUMPS_SHARE -ItemType Directory
-            docker compose up --abort-on-container-exit --no-color
-          displayName: Run HttpStress - HTTP 2.0
-          condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
+        # - powershell: |
+        #     cd '$(httpStressProject)'
+        #     $env:STRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 3.0"
+        #     $env:STRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 3.0"
+        #     New-Item -Force $env:DUMPS_SHARE -ItemType Directory
+        #     docker compose up --abort-on-container-exit --no-color
+        #   displayName: Run HttpStress - HTTP 3.0
+        #   condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
+
+        # - powershell: |
+        #     cd '$(httpStressProject)'
+        #     $env:STRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 2.0"
+        #     $env:STRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 2.0"
+        #     New-Item -Force $env:DUMPS_SHARE -ItemType Directory
+        #     docker compose up --abort-on-container-exit --no-color
+        #   displayName: Run HttpStress - HTTP 2.0
+        #   condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
 
         - powershell: |
             cd '$(httpStressProject)'
-            $env:STRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 1.1"
+            $env:STRESS_CLIENT_ARGS = "-maxExecutionTime 180 -displayInterval 60 -cancelRate 0.5 -http 1.1 -trace"
             $env:STRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 1.1"
             New-Item -Force $env:DUMPS_SHARE -ItemType Directory
             docker compose up --abort-on-container-exit --no-color

src/libraries/System.Net.Http/src/System/Net/Http/HttpClient.cs

@@ -532,7 +532,15 @@ async Task<HttpResponseMessage> Core(
                             responseContentTelemetryStarted = true;
                         }
 
+                        if (NetEventSource.Log.IsEnabled())
+                        {
+                            NetEventSource.Log.HandlerMessage(0, 0, request.GetHashCode(), "HttpClient.SendAsync.Core", "LoadIntoBufferAsync started.");
+                        }
                         await response.Content.LoadIntoBufferAsync(_maxResponseContentBufferSize, cts.Token).ConfigureAwait(false);
+                        if (NetEventSource.Log.IsEnabled())
+                        {
+                            NetEventSource.Log.HandlerMessage(0, 0, request.GetHashCode(), "HttpClient.SendAsync.Core", "LoadIntoBufferAsync finished.");
+                        }
                     }
 
                     return response;

src/libraries/System.Net.Http/src/System/Net/Http/HttpRequestMessage.cs

@@ -19,6 +19,7 @@ public class HttpRequestMessage : IDisposable
         private const int MessageAlreadySent = 1;
         private const int MessageIsRedirect = 2;
         private const int MessageDisposed = 4;
+        private const int MessageCompleted = 8;
 
         // Track whether the message has been sent.
         // The message shouldn't be sent again if this field is equal to MessageAlreadySent.
@@ -176,6 +177,10 @@ public override string ToString()
 
         internal bool WasRedirected() => (_sendStatus & MessageIsRedirect) != 0;
 
+        internal void MarkAsCompleted() => _sendStatus |= MessageCompleted;
+
+        internal bool WasCompleted() => (_sendStatus & MessageCompleted) != 0;
+
         private bool Disposed
         {
             get => (_sendStatus & MessageDisposed) != 0;

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/ChunkedEncodingReadStream.cs

@@ -164,8 +164,8 @@ private async ValueTask<int> ReadAsyncCore(Memory<byte> buffer, CancellationToke
                 // Should only be called if ReadChunksFromConnectionBuffer returned 0.
 
                 Debug.Assert(_connection != null);
-
-                CancellationTokenRegistration ctr = _connection.RegisterCancellation(cancellationToken);
+                HttpConnection connection = _connection;
+                CancellationTokenRegistration ctr = connection.RegisterCancellation(cancellationToken);
                 try
                 {
                     while (true)
@@ -238,6 +238,7 @@ private async ValueTask<int> ReadAsyncCore(Memory<byte> buffer, CancellationToke
                 finally
                 {
                     ctr.Dispose();
+                    if (NetEventSource.Log.IsEnabled()) connection.Trace($"Killed CTR {ctr.GetHashCode()}");
                 }
             }
 
@@ -253,7 +254,8 @@ public override Task CopyToAsync(Stream destination, int bufferSize, Cancellatio
 
             private async Task CopyToAsyncCore(Stream destination, CancellationToken cancellationToken)
             {
-                CancellationTokenRegistration ctr = _connection!.RegisterCancellation(cancellationToken);
+                HttpConnection connection = _connection!;
+                CancellationTokenRegistration ctr = connection.RegisterCancellation(cancellationToken);
                 try
                 {
                     while (true)
@@ -283,6 +285,7 @@ private async Task CopyToAsyncCore(Stream destination, CancellationToken cancell
                 finally
                 {
                     ctr.Dispose();
+                    if (NetEventSource.Log.IsEnabled()) connection.Trace($"Killed CTR {ctr.GetHashCode()}");
                 }
             }
 
@@ -314,6 +317,8 @@ private int ReadChunksFromConnectionBuffer(Span<byte> buffer, CancellationTokenR
             {
                 Debug.Assert(_connection != null);
 
+                HttpConnection connection = _connection;
+
                 try
                 {
                     ReadOnlySpan<byte> currentLine;
@@ -406,6 +411,7 @@ private int ReadChunksFromConnectionBuffer(Span<byte> buffer, CancellationTokenR
                                 // we then return a connection to the pool that has been or will be disposed
                                 // (e.g. if a timer is used and has already queued its callback but the
                                 // callback hasn't yet run).
+                                if (NetEventSource.Log.IsEnabled()) connection.Trace($"Killed CTR {cancellationRegistration.GetHashCode()}");
                                 cancellationRegistration.Dispose();
                                 CancellationHelper.ThrowIfCancellationRequested(cancellationRegistration.Token);
 

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/ConnectionCloseReadStream.cs

@@ -66,6 +66,7 @@ public override async ValueTask<int> ReadAsync(Memory<byte> buffer, Cancellation
                     finally
                     {
                         ctr.Dispose();
+                        if (NetEventSource.Log.IsEnabled()) connection.Trace($"Killed CTR {ctr.GetHashCode()}");
                     }
                 }
 
@@ -126,6 +127,7 @@ private async Task CompleteCopyToAsync(Task copyTask, HttpConnection connection,
                 }
                 finally
                 {
+                    if (NetEventSource.Log.IsEnabled()) connection.Trace($"Killed CTR {ctr.GetHashCode()}");
                     ctr.Dispose();
                 }
 

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/ContentLengthReadStream.cs

@@ -79,6 +79,7 @@ public override async ValueTask<int> ReadAsync(Memory<byte> buffer, Cancellation
                 }
                 else
                 {
+                    HttpConnection connection = _connection;
                     CancellationTokenRegistration ctr = _connection.RegisterCancellation(cancellationToken);
                     try
                     {
@@ -91,6 +92,7 @@ public override async ValueTask<int> ReadAsync(Memory<byte> buffer, Cancellation
                     finally
                     {
                         ctr.Dispose();
+                        if (NetEventSource.Log.IsEnabled()) connection.Trace($"Killed CTR {ctr.GetHashCode()}");
                     }
                 }
 
@@ -144,6 +146,7 @@ public override Task CopyToAsync(Stream destination, int bufferSize, Cancellatio
             private async Task CompleteCopyToAsync(Task copyTask, CancellationToken cancellationToken)
             {
                 Debug.Assert(_connection != null);
+                HttpConnection connection = _connection;
                 CancellationTokenRegistration ctr = _connection.RegisterCancellation(cancellationToken);
                 try
                 {
@@ -156,6 +159,7 @@ private async Task CompleteCopyToAsync(Task copyTask, CancellationToken cancella
                 finally
                 {
                     ctr.Dispose();
+                    if (NetEventSource.Log.IsEnabled()) connection.Trace($"Killed CTR {ctr.GetHashCode()}");
                 }
 
                 Finish();