Introduce the --fail-fast option for requesting VC to fail on first error regardless of severity.

.pipelines/azure-pipelines-linux.yml

@@ -13,7 +13,7 @@ pool:
   vmImage: ubuntu-latest
 
 variables:
-  VcVersion : 1.9.3
+  VcVersion : 1.9.4
   ROOT: $(Build.SourcesDirectory)
   CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning
   ENABLE_PRS_DELAYSIGN: 1

src/VirtualClient/VirtualClient.Contracts/VirtualClientComponent.cs

@@ -106,24 +106,6 @@ protected VirtualClientComponent(IServiceCollection dependencies, IDictionary<st
         /// </summary>
         public IList<Action> CleanupTasks { get; }
 
-        /// <summary>
-        /// Parameter defines the content path format/structure to use when uploading content
-        /// to target storage resources. When not defined the 'Default' structure is used.
-        /// </summary>
-        public string ContentPathFormat
-        {
-            get
-            {
-                this.Parameters.TryGetValue(nameof(this.ContentPathFormat), out IConvertible format);
-                return format?.ToString();
-            }
-
-            set
-            {
-                this.Parameters[nameof(this.ContentPathFormat)] = value;
-            }
-        }
-
         /// <summary>
         /// The CPU/processor architecture (e.g. amd64, arm).
         /// </summary>
@@ -150,6 +132,12 @@ public string ContentPathFormat
         /// </summary>
         public string ExperimentId { get; }
 
+        /// <summary>
+        /// True if VC should exit/crash on first/any error(s) regardless of 
+        /// their severity. Default = false.
+        /// </summary>
+        public bool FailFast { get; set; }
+
         /// <summary>
         /// The client environment/topology layout provided to the Virtual Client application.
         /// </summary>

src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExecutorTests.cs

@@ -687,6 +687,64 @@ public async Task ProfileExecutorCorrelationIdentifiersAreCorrectForMonitorsExec
             }
         }
 
+        [Test]
+        [TestCase(ErrorReason.MonitorFailed)]
+        [TestCase(ErrorReason.WorkloadFailed)]
+        public async Task ProfileExecutorHandlesNonTerminalExceptionsIfTheFailFastOptionIsNotRequested(ErrorReason errorReason)
+        {
+            int iterationsExecuted = 0;
+            using (TestProfileExecutor executor = new TestProfileExecutor(this.mockProfile, this.mockFixture.Dependencies))
+            {
+                executor.ExecuteActions = true;
+                executor.FailFast = false;
+
+                executor.ActionBegin += (sender, args) => throw new WorkloadException($"Expected to be handled", errorReason);
+                executor.IterationEnd += (sender, args) => iterationsExecuted++;
+
+                Task executionTask = executor.ExecuteAsync(new ProfileTiming(profileIterations: 3), CancellationToken.None);
+
+                DateTime testTimeout = DateTime.UtcNow.AddSeconds(10);
+                while (!executionTask.IsCompleted)
+                {
+                    await Task.Delay(10).ConfigureAwait(false);
+                }
+
+                Assert.DoesNotThrow(() => executionTask.ThrowIfErrored());
+                Assert.AreEqual(TaskStatus.RanToCompletion, executionTask.Status);
+                Assert.AreEqual(3, iterationsExecuted);
+            }
+        }
+
+        [Test]
+        [TestCase(ErrorReason.MonitorFailed)]
+        [TestCase(ErrorReason.WorkloadFailed)]
+        [TestCase(ErrorReason.WorkloadDependencyMissing)]
+        public async Task ProfileExecutorExitsImmediatelyOnAnyErrorWheneverTheFailFastOptionIsRequested(ErrorReason errorReason)
+        {
+            int iterationsExecuted = 0;
+            using (TestProfileExecutor executor = new TestProfileExecutor(this.mockProfile, this.mockFixture.Dependencies))
+            {
+                executor.ExecuteActions = true;
+                executor.FailFast = true;
+
+                executor.ActionBegin += (sender, args) => throw new WorkloadException($"Expected to fail on first error", errorReason); 
+                executor.IterationEnd += (sender, args) => iterationsExecuted++;
+
+                Task executionTask = executor.ExecuteAsync(new ProfileTiming(profileIterations: 3), CancellationToken.None);
+
+                DateTime testTimeout = DateTime.UtcNow.AddSeconds(10);
+                while (!executionTask.IsCompleted)
+                {
+                    await Task.Delay(10).ConfigureAwait(false);
+                }
+
+                WorkloadException exception = Assert.Throws<WorkloadException>(() => executionTask.ThrowIfErrored());
+
+                Assert.AreEqual(errorReason, exception.Reason);
+                Assert.AreEqual(1, iterationsExecuted);
+            }
+        }
+
         private class TestProfileExecutor : ProfileExecutor
         {
             public TestProfileExecutor(ExecutionProfile profile, IServiceCollection dependencies, IEnumerable<string> scenarios = null, IDictionary<string, IConvertible> metadata = null, ILogger logger = null)

src/VirtualClient/VirtualClient.Core/ProfileExecutor.cs

@@ -134,6 +134,12 @@ public static int CurrentIteration
         /// </summary>
         public TimeSpan ExitWait { get; set; }
 
+        /// <summary>
+        /// True if VC should exit/crash on first/any error(s) regardless of 
+        /// their severity. Default = false.
+        /// </summary>
+        public bool FailFast { get; set; }
+
         /// <summary>
         /// Logs things to various sources
         /// </summary>
@@ -429,7 +435,7 @@ await this.Logger.LogMessageAsync($"{nameof(ProfileExecutor)}.ExecuteActions", a
                                                 this.ActionEnd?.Invoke(this, new ComponentEventArgs(action));
                                             }
                                         }
-                                        catch (VirtualClientException exc) when ((int)exc.Reason >= 500)
+                                        catch (VirtualClientException exc) when ((int)exc.Reason >= 500 || this.FailFast)
                                         {
                                             // Error reasons have numeric/integer values that indicate their severity. Error reasons
                                             // with a value >= 500 are terminal situations where the workload cannot run successfully
@@ -708,6 +714,7 @@ private List<VirtualClientComponent> CreateComponents(
                     {
                         bool executeComponent = true;
                         VirtualClientComponent runtimeComponent = ComponentFactory.CreateComponent(component, this.Dependencies, this.RandomizationSeed);
+                        runtimeComponent.FailFast = this.FailFast;
 
                         // Metadata: Profile-level (global)
                         if (this.Metadata?.Any() == true)

src/VirtualClient/VirtualClient.Main/OptionFactory.cs

@@ -260,6 +260,26 @@ public static Option CreateExperimentIdOption(bool required = false, object defa
             return option;
         }
 
+        /// <summary>
+        /// Command line option defines whether VC should fail fast on errors.
+        /// </summary>
+        /// <param name="required">Sets this option as required.</param>
+        /// <param name="defaultValue">Sets the default value when none is provided.</param>
+        public static Option CreateFailFastFlag(bool required = true, object defaultValue = null)
+        {
+            Option<bool> option = new Option<bool>(new string[] { "--fail-fast", "--ff" })
+            {
+                Name = "FailFast",
+                Description = "Flag indicates that the application should fail fast and exit immediately on any errors experienced regardless of severity.",
+                ArgumentHelpName = "Flag",
+                AllowMultipleArgumentsPerToken = false,
+            };
+
+            OptionFactory.SetOptionRequirements(option, required, defaultValue);
+
+            return option;
+        }
+
         /// <summary>
         /// An option to set IP address of a Virtual Client API to target/monitor.
         /// </summary>

src/VirtualClient/VirtualClient.Main/Program.cs

@@ -210,7 +210,10 @@ internal static CommandLineBuilder SetupCommandLine(string[] args, CancellationT
                 // --exit-wait
                 OptionFactory.CreateExitWaitOption(required: false, TimeSpan.FromMinutes(30)),
 
-                // --installDependencies
+                // --fail-fast
+                OptionFactory.CreateFailFastFlag(required: false, false),
+
+                // --dependencies
                 OptionFactory.CreateDependenciesFlag(required: false),
 
                 // --iterations

src/VirtualClient/VirtualClient.Main/RunProfileCommand.cs

@@ -43,6 +43,11 @@ internal class RunProfileCommand : CommandBase
         /// </summary>
         public string ExperimentId { get; set; }
 
+        /// <summary>
+        /// True if VC should exit/crash on first/any error(s) regardless of their severity. Default = false.
+        /// </summary>
+        public bool FailFast { get; set; }
+
         /// <summary>
         /// True if the profile dependencies should be installed as the only operations. False if
         /// the profile actions and monitors should also be considered.
@@ -710,6 +715,7 @@ await this.CaptureSystemInfoAsync(dependencies, cancellationToken)
                 profileExecutor.ExecuteActions = false;
                 profileExecutor.ExecuteMonitors = false;
                 profileExecutor.ExitWait = this.ExitWait;
+                profileExecutor.FailFast = this.FailFast;
 
                 profileExecutor.BeforeExiting += (source, args) =>
                 {
@@ -779,6 +785,7 @@ await this.CaptureSystemInfoAsync(dependencies, cancellationToken)
             {
                 profileExecutor.RandomizationSeed = this.RandomizationSeed;
                 profileExecutor.ExitWait = this.ExitWait;
+                profileExecutor.FailFast = this.FailFast;
 
                 profileExecutor.BeforeExiting += (source, args) =>
                 {

src/VirtualClient/VirtualClient.UnitTests/CommandLineOptionTests.cs

@@ -65,6 +65,7 @@ public void VirtualClientDefaultCommandRequiresTheProfileOptionBeSupplied()
         [TestCase("--contentPath", "anyname1/anyname2/{experimentId}/{agentId}/anyname3/{toolName}/{role}/{scenario}")]
         [TestCase("--cspt", "anyname1/anyname2/{experimentId}/{agentId}/anyname3/{toolName}/{role}/{scenario}")]
         [TestCase("--debug", null)]
+        [TestCase("--verbose", null)]
         [TestCase("--dependencies", null)]
         [TestCase("--eventHubConnectionString", "ConnectionString")]
         [TestCase("--eventhubconnectionstring", "ConnectionString")]
@@ -75,7 +76,10 @@ public void VirtualClientDefaultCommandRequiresTheProfileOptionBeSupplied()
         [TestCase("--experimentid", "0B692DEB-411E-4AC1-80D5-AF539AE1D6B2")]
         [TestCase("--experiment", "0B692DEB-411E-4AC1-80D5-AF539AE1D6B2")]
         [TestCase("--e", "0B692DEB-411E-4AC1-80D5-AF539AE1D6B2")]
+        [TestCase("--fail-fast", null)]
+        [TestCase("--ff", null)]
         [TestCase("--flush-wait", "00:10:00")]
+        [TestCase("--exit-wait", "00:10:00")]
         [TestCase("--fw", "00:10:00")]
         [TestCase("--i", "3")]
         [TestCase("--iterations", "3")]

src/VirtualClient/VirtualClient.UnitTests/OptionFactoryTests.cs

@@ -192,6 +192,7 @@ public void ContentPathPatternOptionSupportsExpectedAliases(string alias)
 
         [Test]
         [TestCase("--debug")]
+        [TestCase("--verbose")]
         public void DebugFlagSupportsExpectedAliases(string alias)
         {
             Option option = OptionFactory.CreateDebugFlag();
@@ -246,6 +247,16 @@ public void ExitWaitOptionValueMustBeAValidTimeSpanOrIntegerFormat()
             Assert.DoesNotThrow(() => option.Parse("--exit-wait=1440"));
         }
 
+        [Test]
+        [TestCase("--fail-fast")]
+        [TestCase("--ff")]
+        public void FailFastFlagSupportsExpectedAliases(string alias)
+        {
+            Option option = OptionFactory.CreateFailFastFlag();
+            ParseResult result = option.Parse(alias);
+            Assert.IsFalse(result.Errors.Any());
+        }
+
         [Test]
         [TestCase("--ipAddress")]
         [TestCase("--ipaddress")]

website/docs/guides/0010-command-line.md

@@ -15,6 +15,7 @@ on the system.
 | --cspt, --contentPath, --contentPathPattern=\<folderPattern\>  | No       | string/text                  | The content path format/structure to use when uploading content to target storage resources. When not defined the 'Default' structure is used. Default: "{experimentId}/{agentId}/{toolName}/{role}/{scenario}" |
 | --eh, --eventHub, --eventHubConnectionString=\<accesspolicy\>  | No       | string/connection string     | A full connection string/access policy for the Azure Event Hub namespace where telemetry should be written. Contact the VC Team to get an access policy for your team. See [Azure Event Hub Integration](./0610-integration-event-hub.md). |
 | --e, --experimentId=\<guid\>                                   | No       | guid                         | A unique identifier that defines the ID of the experiment for which the Virtual Client workload is associated. |
+| --ff, --fail-fast                                              | No       |                              | Flag indicates that the application should exit immediately on first/any errors regardless of their severity. This applies to 'Actions' in the profile only. 'Dependencies' are ALWAYS implemented to fail fast. 'Monitors' are generally implemented to handle transient issues and to keep running/trying in the background.  |
 | --lp, --layoutPath=\<path\>                                    | No       | string/path                  | A path to a environment layout file that provides additional metadata about the system/hardware on which the Virtual Client will run and information required to support client/server advanced topologies. See [Client/Server Support](./0020-client-server.md). |
 | --ltf, --log-to-file, --logtofile                              | No       |                              | Flag indicates that the output of processes executed by the Virtual Client should be written to log files in the logs directory. |
 | --mt, --metadata=\<key=value,,,key=value...\>                  | No       | string/text                  | Metadata to include with all logs/telemetry output from the Virtual Client. <br/><br/>Each metadata entry should be akey value pair separated by ",,," delimiters (e.g. property1=value1,,,property2=value2). |

website/docs/guides/0200-usage-examples.md

@@ -60,10 +60,20 @@ the application can be correlated with the data captured by the automation syste
 will be included in ALL metrics, counters, logs etc... telemetry that is emitted by the application.
 
 ```
-VirtualClient.exe --profile=PERF-CPU-COREMARK.json --timeout=03:00:00 --packages="{BlobStoreConnectionString|SAS URI}" --metadata="experimentGroup=Group A,,,nodeId=eB3fc2d9-157b-4efc-b39c-a454a0779a5b,,,tipSessionId=73e8ae54-e0a0-48b6-9bda-4a269672b9b1,,,cluster=cluster01,,,region=East US 2"
+VirtualClient.exe --profile=PERF-CPU-GEEKBENCH.json --timeout=03:00:00 --packages="{BlobStoreConnectionString|SAS URI}" --metadata="experimentGroup=Group A,,,nodeId=eB3fc2d9-157b-4efc-b39c-a454a0779a5b,,,tipSessionId=73e8ae54-e0a0-48b6-9bda-4a269672b9b1,,,cluster=cluster01,,,region=East US 2"
 ```
 
-## Scenario: Log Process Output to the File System
+## Scenario: Instruct the Application to Fail Fast
+Virtual Client typically will continue to retry the execution of actions within a profile in the event that one of the actions fails
+for a non-terminal reason. Users may want to instruct the application to promptly exist on any errors regardless of the severity (terminal or not).
+Note that this generally refers to 'Actions' in the profile. The application always fails fast on the failure of 'Dependencies'. 'Monitors' are
+typically implemented to handle exceptions due to the requirement they continue to operate in the background even on failure.
+
+```
+VirtualClient.exe --profile=PERF-CPU-GEEKBENCH.json --timeout=03:00:00 --packages="{BlobStoreConnectionString|SAS URI}" --fail-fast
+```
+
+## Scenario: Write the Output of Processes to the File System
 Virtual Client runs a wide range of workloads, monitors and dependency handlers when executing a given profile. The following examples show
 how to instruct the application to log the output of processes to files in the logs directory on the file system.