Fix some failing ATH tests, increase some timeout to avoid flakyness …

…of some tests (#2495) Fix some failing ATH tests, increase some timeout to avoid flakyness of some tests --------- Signed-off-by: Olivier Lamy <olamy@apache.org>
jenkinsci · Sep 16, 2023 · 76bc08e · 76bc08e
1 parent 65d09d0
commit 76bc08e
Show file tree

Hide file tree

Showing 19 changed files with 119 additions and 24 deletions.
diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml
@@ -1,4 +1,10 @@
 <extensions xmlns="http://maven.apache.org/EXTENSIONS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/EXTENSIONS/1.0.0 http://maven.apache.org/xsd/core-extensions-1.0.0.xsd">
+<!--  cache doesn't like incremental so disable it for now... -->
+<!--  <extension>-->
+<!--    <groupId>org.apache.maven.extensions</groupId>-->
+<!--    <artifactId>maven-build-cache-extension</artifactId>-->
+<!--    <version>1.0.1</version>-->
+<!--  </extension>  -->
   <extension>
     <groupId>io.jenkins.tools.incrementals</groupId>
     <artifactId>git-changelist-maven-extension</artifactId>

diff --git a/.mvn/maven.config b/.mvn/maven.config
@@ -1,2 +1,3 @@
 -Pconsume-incrementals
 -Pmight-produce-incrementals
+-Daether.dependencyCollector.impl=bf
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -65,7 +65,7 @@ node() {
           stage('Building BlueOcean') {
             timeout(time: 90, unit: 'MINUTES') {
               try {
-                sh "mvn clean install -T1C -V -B -DcleanNode -ntp -Dmaven.test.failure.ignore -s settings.xml -e -Dmaven.artifact.threads=30"
+                sh "mvn clean install -T2 -Pci -V -B -DcleanNode -ntp -DforkCount=3 -Dmaven.test.failure.ignore -s settings.xml -e -Dmaven.artifact.threads=30"
               } finally {
                 junit testResults: '**/target/surefire-reports/TEST-*.xml', allowEmptyResults: true
                 junit testResults: '**/target/jest-reports/*.xml', allowEmptyResults: true
@@ -76,9 +76,9 @@ node() {
 
           jenkinsVersions.each { version ->
             stage("ATH - Jenkins ${version}") {
-              timeout(time: 150, unit: 'MINUTES') {
+              timeout(time: 90, unit: 'MINUTES') {
                 dir('acceptance-tests') {
-                  sh "bash -x ./run.sh -v=${version} --host=${ip} --no-selenium --settings='-s ${env.WORKSPACE}/settings.xml'"
+                  sh "bash -x ./run.sh -v=${version} --host=${ip} --no-selenium -ci --settings='-s ${env.WORKSPACE}/settings.xml'"
                   junit '**/target/surefire-reports/*.xml'
                   archive '**/target/screenshots/**/*'
                 }

diff --git a/acceptance-tests/pom.xml b/acceptance-tests/pom.xml
@@ -14,6 +14,7 @@
         <argLine>-Djdk.net.URLClassPath.disableClassPathURLCheck=true</argLine>
         <slf4j.version>1.7.32</slf4j.version>
         <selenium.version>3.141.59</selenium.version> <!-- 3.141.59 4.0.0-alpha-6 -->
+        <webDriverDefaultTimeout>180000</webDriverDefaultTimeout>
     </properties>
 
 
@@ -287,5 +288,26 @@
                 </plugins>
             </build>
         </profile>
+        <profile>
+            <id>ci</id>
+            <build>
+                <pluginManagement>
+                    <plugins>
+                        <plugin>
+                            <groupId>org.apache.maven.plugins</groupId>
+                            <artifactId>maven-surefire-plugin</artifactId>
+                            <configuration>
+                                <excludedGroups>flaky</excludedGroups>
+                                <systemPropertyVariables>
+                                    <ath.defaultRetry>4</ath.defaultRetry>
+                                    <webDriverDefaultTimeout>240000</webDriverDefaultTimeout>
+                                </systemPropertyVariables>
+<!--                                <rerunFailingTestsCount>2</rerunFailingTestsCount>-->
+                            </configuration>
+                        </plugin>
+                    </plugins>
+                </pluginManagement>
+            </build>
+        </profile>
     </profiles>
 </project>
diff --git a/acceptance-tests/runner/runtime/pom.xml b/acceptance-tests/runner/runtime/pom.xml
@@ -25,6 +25,7 @@
     <argLine>-Djdk.net.URLClassPath.disableClassPathURLCheck=true</argLine>
     <java.level>8</java.level>
     <slf4j.version>1.7.30</slf4j.version>
+    <webDriverDefaultTimeout>120000</webDriverDefaultTimeout>
   </properties>
 
   <repositories>
@@ -242,6 +243,27 @@
         </plugins>
       </build>
     </profile>
+    <profile>
+      <id>ci</id>
+      <build>
+        <pluginManagement>
+          <plugins>
+            <plugin>
+              <groupId>org.apache.maven.plugins</groupId>
+              <artifactId>maven-surefire-plugin</artifactId>
+              <configuration>
+                <excludedGroups>flaky</excludedGroups>
+                <systemPropertyVariables>
+                <ath.defaultRetry>4</ath.defaultRetry>
+                <webDriverDefaultTimeout>${webDriverDefaultTimeout}</webDriverDefaultTimeout>
+                </systemPropertyVariables>
+                <!--                                <rerunFailingTestsCount>2</rerunFailingTestsCount>-->
+                </configuration>
+            </plugin>
+          </plugins>
+        </pluginManagement>
+      </build>
+    </profile>
   </profiles>
 
   <build>

diff --git a/acceptance-tests/runner/scripts/args.sh b/acceptance-tests/runner/scripts/args.sh
@@ -11,7 +11,7 @@ PLUGINS=""
 AGGREGATOR_DIR=""
 DEV_JENKINS=false
 PROFILES="-P runTests"
-JENKINS_JAVA_OPTS="-Djava.util.logging.config.file=./logging.properties -Dio.jenkins.blueocean.rest.impl.pipeline.credential.BlueOceanCredentialsProvider.enabled=true"
+JENKINS_JAVA_OPTS="-Djava.util.logging.config.file=./logging.properties -Dhudson.plugins.git.GitSCM.ALLOW_LOCAL_CHECKOUT=true -Dio.jenkins.blueocean.rest.impl.pipeline.credential.BlueOceanCredentialsProvider.enabled=true"
 TEST_TO_RUN=""
 
 for i in "$@"
@@ -32,6 +32,9 @@ case $i in
     -ld|--local-dev)
     export LOCAL_DEV=true
     ;;
+    -ci|--ci-bo)
+    export CI_BO=true
+    ;;
     --no-selenium)
     RUN_SELENIUM=false
     ;;
@@ -77,6 +80,12 @@ if [ "${DEV_JENKINS}" == "true" ]; then
     TEST_TO_RUN=""
 fi
 
+if [ "${CI_BO}" == "true" ]; then
+    # Use ci profile which can exclude some flaky tests
+    PROFILES="-P runTests -P ci"
+    TEST_TO_RUN=""
+fi
+
 
 # For now, the location of the aggregator plugin must be defined until we have
 # blueocean plugins in the Update Center.

diff --git a/acceptance-tests/src/main/java/io/blueocean/ath/ATHJUnitRunner.java b/acceptance-tests/src/main/java/io/blueocean/ath/ATHJUnitRunner.java
@@ -172,7 +172,8 @@ private void runTest(Statement statement, Description description,
         eachNotifier.fireTestStarted();
         List<Throwable> failures = new ArrayList<>();
         try {
-            int n = retry == null ? 1 : retry.value();
+            int defaultRetry = Integer.getInteger("ath.defaultRetry",1);
+            int n = retry == null ? defaultRetry : Math.max(retry.value(), defaultRetry);
 
             for (int i = 0; i < n; i++) {
                 try {

diff --git a/acceptance-tests/src/main/java/io/blueocean/ath/SmartWebElement.java b/acceptance-tests/src/main/java/io/blueocean/ath/SmartWebElement.java
@@ -27,7 +27,7 @@
  */
 public class SmartWebElement implements WebElement {
     private static Logger logger = LoggerFactory.getLogger(SmartWebElement.class);
-    public static final int DEFAULT_TIMEOUT = Integer.getInteger("webDriverDefaultTimeout", 3000);
+    public static final int DEFAULT_TIMEOUT = WaitUtil.DEFAULT_TIMEOUT;
     public static final int RETRY_COUNT = 3;
 
     private WebDriver driver;

diff --git a/acceptance-tests/src/main/java/io/blueocean/ath/WaitUtil.java b/acceptance-tests/src/main/java/io/blueocean/ath/WaitUtil.java
@@ -21,7 +21,7 @@
 
 @Singleton
 public class WaitUtil {
-    public static int DEFAULT_TIMEOUT = Integer.getInteger("webDriverDefaultTimeout", 20000);
+    public static int DEFAULT_TIMEOUT = Integer.getInteger("webDriverDefaultTimeout", 30000);
     public static String DEFAULT_ERROR_MESSAGE = "Error while waiting for something";
     public static final int RETRY_COUNT = 10;
 

diff --git a/acceptance-tests/src/main/java/io/blueocean/ath/pages/blue/ActivityPage.java b/acceptance-tests/src/main/java/io/blueocean/ath/pages/blue/ActivityPage.java
@@ -62,13 +62,13 @@ public void checkPipeline() {
 
     public ActivityPage checkUrl() {
         wait.until(ExpectedConditions.urlContains(pipeline.getUrl() + "/activity"), 120000);
-        wait.until(By.cssSelector("article.activity"), 60000);
+        wait.until(By.cssSelector("article.activity"));
         return this;
     }
 
     public ActivityPage checkUrl(String filter) {
         wait.until(ExpectedConditions.urlContains(pipeline.getUrl() + "/activity?branch=" + URLEncoder.encode(URLEncoder.encode(filter))), 30000);
-        wait.until(By.cssSelector("article.activity"), 60000);
+        wait.until(By.cssSelector("article.activity"));
         return this;
     }
 
@@ -134,7 +134,7 @@ public void testNumberRunsComplete(int atLeast, String status) {
 
     public void checkBasicDomElements() {
         wait.retryAction("check that we are on the activity page", 3, driver -> {
-            wait.until(By.cssSelector("article.activity"), 5000);
+            wait.until(By.cssSelector("article.activity"));
             logger.info("checkBasicDomElements: Activity tab found");
             return true;
         });

diff --git a/acceptance-tests/src/test/java/io/blueocean/ath/live/GithubEditorTest.java b/acceptance-tests/src/test/java/io/blueocean/ath/live/GithubEditorTest.java
@@ -3,6 +3,7 @@
 import io.blueocean.ath.ATHJUnitRunner;
 import io.blueocean.ath.CustomJenkinsServer;
 import io.blueocean.ath.Login;
+import io.blueocean.ath.Retry;
 import io.blueocean.ath.factory.MultiBranchPipelineFactory;
 import io.blueocean.ath.model.MultiBranchPipeline;
 import io.blueocean.ath.pages.blue.ActivityPage;
@@ -75,6 +76,7 @@ public void createBlankRepo() throws IOException {
      *
      * Creates a blank github repo, and then uses editor to create a simple pipeline.
      */
+    @Retry(3)
     @Test
     public void testEditor() throws IOException {
         creationPage.createPipeline(helper.getAccessToken(), helper.getOrganizationOrUsername(), helper.getActualRepositoryName(), true);
@@ -96,6 +98,7 @@ public void testEditor() throws IOException {
     /**
      * This test covers creation of a pipeline, and changes agent settings within it.
      */
+    @Retry(3)
     @Test
     public void testEditorChangeAgentSetting() throws IOException {
         String newBranchName = "made-by-testEditorChangeAgentSetting";
@@ -120,6 +123,7 @@ public void testEditorChangeAgentSetting() throws IOException {
      * stage within that same pipeline, then saves it to a new branch.
      */
     @Test
+    @Retry(3)
     public void testEditorAddAndDeleteStage() throws IOException {
         String firstBranchName = "branch-before-delete";
         String secondBranchName = "branch-after-delete";
@@ -153,6 +157,7 @@ public void testEditorAddAndDeleteStage() throws IOException {
      * This test covers creation of a pipeline, and adds an environment
      * variable to it.
      */
+    @Retry(3)
     @Test
     public void testEditorSetEnvironmentVariables() throws IOException {
         String newBranchName = "made-by-testEditorSetEnvironmentVariables";
@@ -175,6 +180,7 @@ public void testEditorSetEnvironmentVariables() throws IOException {
     /**
      * Make sure we can paste a bad token that has whitespace added.
      */
+    @Retry(3)
     @Test
     public void testEditorWithSpace() throws IOException {
         // Gotta make Jenkins clear out its credential store or we might get a false positive depending on test order
@@ -187,6 +193,7 @@ public void testEditorWithSpace() throws IOException {
      *
      * Creates a blank github repo, and then uses editor to create a parallel pipeline.
      */
+    @Retry(3)
     @Test
     public void testEditorParallel() throws IOException {
         String branchNameForParallelPipeline = "branch-with-parallels";

diff --git a/acceptance-tests/src/test/java/io/blueocean/ath/offline/edgeCases/FolderTest.java b/acceptance-tests/src/test/java/io/blueocean/ath/offline/edgeCases/FolderTest.java
@@ -2,11 +2,7 @@
 
 import com.mashape.unirest.http.exceptions.UnirestException;
 import com.offbytwo.jenkins.model.FolderJob;
-import io.blueocean.ath.ATHJUnitRunner;
-import io.blueocean.ath.BaseUrl;
-import io.blueocean.ath.BlueOceanAcceptanceTest;
-import io.blueocean.ath.GitRepositoryRule;
-import io.blueocean.ath.WaitUtil;
+import io.blueocean.ath.*;
 import io.blueocean.ath.api.classic.ClassicJobApi;
 import io.blueocean.ath.factory.MultiBranchPipelineFactory;
 import io.blueocean.ath.model.Folder;
@@ -85,6 +81,7 @@ public void tearDown() throws IOException {
      * As long as activity loads run, any other page for this pipeline should load as it uses a shared router.
      */
     @Test
+    @Retry(3)
     public void multiBranchFolderTest() throws GitAPIException, IOException {
         String pipelineName = "FolderTest_multiBranchFolderTest";
         git.writeJenkinsFile(loadJenkinsFile());
@@ -100,6 +97,7 @@ public void multiBranchFolderTest() throws GitAPIException, IOException {
     }
 
     @Test
+    @Retry(3)
     public void foldersTest() throws IOException, GitAPIException, UnirestException, InterruptedException {
         String pipelineName = "Sohn";
 
@@ -109,7 +107,7 @@ public void foldersTest() throws IOException, GitAPIException, UnirestException,
 
         jobApi.createFreeStyleJob(folderJob, pipelineName, "echo 'hello world!'");
         driver.get(folderJob.getUrl()+"/job/"+pipelineName+"/");
-        driver.findElement(By.xpath("//a[contains(@class, 'task-link') and @title='Open Blue Ocean']")).click();
+        driver.findElement(By.xpath("//a[contains(@class,'task-link')][span[contains(@class,'task-link-text') and contains(text(),'Open Blue Ocean')]]")).click();
         wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector(".PlaceholderContent.NoRuns")));
         String activityPage = driver.getCurrentUrl();
         assertTrue(activityPage.endsWith(getNestedPipelinePath("firstFolder") +
@@ -119,7 +117,7 @@ public void foldersTest() throws IOException, GitAPIException, UnirestException,
         wait.until(By.cssSelector("nav a.pr"));
         wait.until(By.cssSelector("a.main_exit_to_app"));
         driver.findElement(By.cssSelector("a.main_exit_to_app")).click();
-        wait.until(By.xpath("//a[contains(@class, 'task-link') and @title='Open Blue Ocean']"));
+        wait.until(By.xpath("//a[contains(@class,'task-link')][span[contains(@class,'task-link-text') and contains(text(),'Open Blue Ocean')]]"));
         assertEquals(base+"/job/firstFolder/job/"+
                 URLEncoder.encode("三百", "UTF-8")
                 +"/job/"+URLEncoder.encode("ñba", "UTF-8")
@@ -128,6 +126,7 @@ public void foldersTest() throws IOException, GitAPIException, UnirestException,
     }
 
     @Test
+    @Retry(3)
     public void anotherFoldersTest() throws IOException, GitAPIException, UnirestException {
         Folder anotherFolder =  Folder.folders("anotherFolder", "三百", "ñba", "七");
         FolderJob folderJob = jobApi.createFolders(anotherFolder, true);
@@ -200,7 +199,7 @@ public void anotherFoldersTest() throws IOException, GitAPIException, UnirestExc
         // make sure the open blue ocean button works. In this case,
         // it should bring the browser to the run details page for the first run
         driver.get(base+"/job/anotherFolder/job/三百/job/ñba/job/七/job/"+pipelineName+"/job/feature%252F1/1/");
-        wait.until(By.xpath("//a[contains(@class, 'task-link') and @title='Open Blue Ocean']")).click();
+        wait.until(By.xpath("//a[contains(@class,'task-link')][span[contains(@class,'task-link-text') and contains(text(),'Open Blue Ocean')]]")).click();
         wait.until(ExpectedConditions.urlContains(getNestedPipelinePath("anotherFolder") + pipelineName + "/detail/feature%2F1/1/pipeline"));
         wait.until(ExpectedConditions.or(
                 ExpectedConditions.presenceOfElementLocated(By.cssSelector(".RunDetails-content .log-wrapper")),
@@ -211,7 +210,7 @@ public void anotherFoldersTest() throws IOException, GitAPIException, UnirestExc
         // it should bring the browser to the main top-level pipelines page.
         // See https://issues.jenkins-ci.org/browse/JENKINS-39842
         driver.get(base+"/job/anotherFolder/job/三百/job/ñba");
-        wait.until(By.xpath("//a[contains(@class, 'task-link') and @title='Open Blue Ocean']")).click();
+        wait.until(By.xpath("//a[contains(@class,'task-link')][span[contains(@class,'task-link-text') and contains(text(),'Open Blue Ocean')]]")).click();
 
         wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector(".Header-topNav nav a[href=\"/blue/pipelines\"]")));
         driver.findElement(By.cssSelector(".pipelines-table"));

diff --git a/acceptance-tests/src/test/java/io/blueocean/ath/offline/edgeCases/TestDetailsTest.java b/acceptance-tests/src/test/java/io/blueocean/ath/offline/edgeCases/TestDetailsTest.java
@@ -4,6 +4,7 @@
 import io.blueocean.ath.ATHJUnitRunner;
 import io.blueocean.ath.BlueOceanAcceptanceTest;
 import io.blueocean.ath.GitRepositoryRule;
+import io.blueocean.ath.Retry;
 import io.blueocean.ath.factory.MultiBranchPipelineFactory;
 import io.blueocean.ath.model.MultiBranchPipeline;
 import io.blueocean.ath.pages.blue.RunDetailsTestsPage;
@@ -45,6 +46,7 @@ public void setup() throws IOException, GitAPIException {
     }
 
     @Test
+    @Retry(3)
     public void testTests(){
         RunDetailsTestsPage runDetailsTestsPage = pipeline.getRunDetailsTestsPage().open("master", 1);
         runDetailsTestsPage.checkResults("success", 10);

diff --git a/acceptance-tests/src/test/java/io/blueocean/ath/offline/multibranch/CommitMessagesTest.java b/acceptance-tests/src/test/java/io/blueocean/ath/offline/multibranch/CommitMessagesTest.java
@@ -3,6 +3,7 @@
 import io.blueocean.ath.ATHJUnitRunner;
 import io.blueocean.ath.BlueOceanAcceptanceTest;
 import io.blueocean.ath.GitRepositoryRule;
+import io.blueocean.ath.Retry;
 import io.blueocean.ath.api.classic.ClassicJobApi;
 import io.blueocean.ath.factory.MultiBranchPipelineFactory;
 import io.blueocean.ath.model.MultiBranchPipeline;
@@ -58,6 +59,7 @@ public void tearDown() throws IOException {
      * This tests the commit messages are being picked up from git and displayed on the run in activity.
      */
     @Test
+    @Retry(3)
     public void commitMessagesTest() throws IOException, GitAPIException {
         final String pipelineName = this.getClass().getSimpleName() + "_" + name.getMethodName();
         final String branchName = "master";

diff --git a/...ance-tests/src/test/java/io/blueocean/ath/offline/multibranch/ParallelNavigationTest.java b/...ance-tests/src/test/java/io/blueocean/ath/offline/multibranch/ParallelNavigationTest.java
@@ -93,6 +93,7 @@ public void parallelNavigationTest() throws IOException, GitAPIException, Interr
      * One at a time, the proceed button will be clicked.
      */
     @Test
+    @Retry(3)
     public void parallelNavigationTestInput() throws IOException, GitAPIException, InterruptedException {
         // Create navTestWithInput
         logger.info("Creating pipeline {}", navTestWithInput);
@@ -123,6 +124,7 @@ public void parallelNavigationTestInput() throws IOException, GitAPIException, I
      * This checks that an error is shown in the UI for a failed input step
      */
     @Test
+    @Retry(3)
     public void failedInputStep() throws IOException, GitAPIException, InterruptedException {
         try
         {
@@ -157,6 +159,7 @@ public void failedInputStep() throws IOException, GitAPIException, InterruptedEx
      * This checks that the log is visible when a run fails with no steps or stages
      */
     @Test
+    @Retry(3)
     public void testLogVisibilityWhenNoStepsOrStages() throws IOException, GitAPIException, InterruptedException {
         // Create navTestWithNoStepsNoStages
         logger.info("Creating pipeline " + navTestWithNoStepsNoStages);
@@ -180,6 +183,7 @@ public void testLogVisibilityWhenNoStepsOrStages() throws IOException, GitAPIExc
      * This checks that href attr of the alert error for unsupported inputs leads to the correct classic url
      */
     @Test
+    @Retry(3)
     public void testInputParamGoToClassicLink() throws IOException, GitAPIException, InterruptedException {
         // Create navTestInputParamGoToClassicLink
         logger.info("Creating pipeline " + navTestInputParamGoToClassicLink);