diff --git a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/CompilerInstallationTests.cs b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/CompilerInstallationTests.cs
index 7df173081..3b2b9f566 100644
--- a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/CompilerInstallationTests.cs
+++ b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/CompilerInstallationTests.cs
@@ -77,6 +77,7 @@ public async Task CompilerInstallationRunsTheExpectedWorkloadCommandInLinuxForGc
                     $"--slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-123 " +
                     $"--slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-123 " +
                     $"--slave /usr/bin/gfortran gfortran /usr/bin/gfortran-123",
+                "sudo update-alternatives --remove-all cpp",
                 "sudo update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-123 1230",
             };
 
@@ -109,7 +110,7 @@ public async Task CompilerInstallationRunsTheExpectedWorkloadCommandInLinuxForGc
                 await compilerInstallation.ExecuteAsync(CancellationToken.None).ConfigureAwait(false);
             }
 
-            Assert.AreEqual(7, commandExecuted);
+            Assert.AreEqual(expectedCommands.Count(), commandExecuted);
         }
 
         [Test]
@@ -252,6 +253,7 @@ public async Task CompilerInstallationInLinuxDefaultsToGcc10()
                     $"--slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-10 " +
                     $"--slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-10 " +
                     $"--slave /usr/bin/gfortran gfortran /usr/bin/gfortran-10",
+                "sudo update-alternatives --remove-all cpp",
                 "sudo update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-10 100",
             };
 
@@ -285,7 +287,7 @@ public async Task CompilerInstallationInLinuxDefaultsToGcc10()
                 await compilerInstallation.ExecuteAsync(CancellationToken.None).ConfigureAwait(false);
             }
 
-            Assert.AreEqual(7, commandExecuted);
+            Assert.AreEqual(expectedCommands.Count(), commandExecuted);
         }
 
         [Test]
diff --git a/src/VirtualClient/VirtualClient.Dependencies/CompilerInstallation.cs b/src/VirtualClient/VirtualClient.Dependencies/CompilerInstallation.cs
index f13696483..1d52295b0 100644
--- a/src/VirtualClient/VirtualClient.Dependencies/CompilerInstallation.cs
+++ b/src/VirtualClient/VirtualClient.Dependencies/CompilerInstallation.cs
@@ -274,6 +274,9 @@ private async Task SetGccPriorityAsync(string gccVersion, EventContext telemetry
 
             await this.ExecuteCommandAsync("update-alternatives", updateAlternativeArgument, Environment.CurrentDirectory, telemetryContext, cancellationToken);
 
+            // Remove all existing alternatives for cpp before the subsequent "update-alternatives" of cpp
+            await this.ExecuteCommandAsync("update-alternatives", "--remove-all cpp", Environment.CurrentDirectory, telemetryContext, cancellationToken);
+
             // For some update path, the cpp can't be update-alternative by a gcc, so needs a separate call.
             string updateAlternativeArgumentCpp = $"--install /usr/bin/cpp cpp /usr/bin/cpp-{gccVersion} {gccVersion}0";