[bazel] Mark remaining tests as being skipped for remote builds

.bazelrc

@@ -136,3 +136,6 @@ build:remote-ci --bes_upload_mode=wait_for_upload_complete
 # make reasonable usage of everything, everywhere, all at once.
 build:remote --local_cpu_resources='HOST_CPUS*10'
 build:remote --local_ram_resources='HOST_RAM*4.0'
+
+# A small hint that we're running our tests remotely
+test:remote --test_env=REMOTE_BUILD=1

java/private/selenium_test.bzl

@@ -77,6 +77,7 @@ def selenium_test(name, test_class, size = "medium", browsers = BROWSERS.keys(),
     stripped_args.pop("data", None)
     stripped_args.pop("jvm_flags", None)
     stripped_args.pop("tags", None)
+    inherited_env = stripped_args.pop("env_inherit", []) + ["REMOTE_BUILD"]
 
     all_tests = []
 
@@ -94,6 +95,7 @@ def selenium_test(name, test_class, size = "medium", browsers = BROWSERS.keys(),
             # Only allow linting on the default test
             tags = BROWSERS[browser]["tags"] + tags + ([] if test == name else ["no-lint"]),
             data = BROWSERS[browser]["data"] + data,
+            env_inherit = inherited_env,
             **stripped_args
         )
         if browser == default_browser:

java/test/org/openqa/selenium/ClickTest.java

@@ -17,15 +17,6 @@
 
 package org.openqa.selenium;
 
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.openqa.selenium.WaitingConditions.newWindowIsOpened;
-import static org.openqa.selenium.WaitingConditions.pageSourceToContain;
-import static org.openqa.selenium.support.ui.ExpectedConditions.titleIs;
-import static org.openqa.selenium.testing.drivers.Browser.FIREFOX;
-import static org.openqa.selenium.testing.drivers.Browser.IE;
-import static org.openqa.selenium.testing.drivers.Browser.SAFARI;
-
-import java.util.Set;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.openqa.selenium.testing.Ignore;
@@ -34,6 +25,16 @@
 import org.openqa.selenium.testing.NotYetImplemented;
 import org.openqa.selenium.testing.SwitchToTopAfterTest;
 
+import java.util.Set;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.openqa.selenium.WaitingConditions.newWindowIsOpened;
+import static org.openqa.selenium.WaitingConditions.pageSourceToContain;
+import static org.openqa.selenium.support.ui.ExpectedConditions.titleIs;
+import static org.openqa.selenium.testing.drivers.Browser.FIREFOX;
+import static org.openqa.selenium.testing.drivers.Browser.IE;
+import static org.openqa.selenium.testing.drivers.Browser.SAFARI;
+
 class ClickTest extends JupiterTestBase {
 
   @BeforeEach

java/test/org/openqa/selenium/CookieImplementationTest.java

@@ -20,6 +20,8 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 import static org.openqa.selenium.testing.drivers.Browser.ALL;
+import static org.openqa.selenium.testing.drivers.Browser.CHROME;
+import static org.openqa.selenium.testing.drivers.Browser.FIREFOX;
 import static org.openqa.selenium.testing.drivers.Browser.IE;
 import static org.openqa.selenium.testing.drivers.Browser.SAFARI;
 
@@ -32,6 +34,7 @@
 import org.openqa.selenium.environment.DomainHelper;
 import org.openqa.selenium.testing.Ignore;
 import org.openqa.selenium.testing.JupiterTestBase;
+import org.openqa.selenium.testing.NotWorkingInRemoteBazelBuilds;
 import org.openqa.selenium.testing.NotYetImplemented;
 import org.openqa.selenium.testing.SwitchToTopAfterTest;
 
@@ -81,6 +84,7 @@ public void testShouldGetCookieByName() {
 
   @Test
   @NotYetImplemented(SAFARI)
+  @NotWorkingInRemoteBazelBuilds(FIREFOX)
   public void testShouldBeAbleToAddCookie() {
     String key = generateUniqueKey();
     String value = "foo";
@@ -200,6 +204,8 @@ public void testAddCookiesWithDifferentPathsThatAreRelatedToOurs() {
   @SwitchToTopAfterTest
   @Test
   @Ignore(SAFARI)
+  @NotWorkingInRemoteBazelBuilds(CHROME)
+  @NotWorkingInRemoteBazelBuilds(FIREFOX)
   public void testGetCookiesInAFrame() {
     driver.get(domainHelper.getUrlForFirstValidHostname("/common/animals"));
     Cookie cookie1 = new Cookie.Builder("fish", "cod").path("/common/animals").build();
@@ -239,6 +245,7 @@ public void testShouldNotGetCookieOnDifferentDomain() {
 
   @Test
   @NotYetImplemented(SAFARI)
+  @NotWorkingInRemoteBazelBuilds(FIREFOX)
   public void testShouldBeAbleToAddToADomainWhichIsRelatedToTheCurrentDomain() {
     String cookieName = "name";
     assertCookieIsNotPresentWithName(cookieName);
@@ -263,6 +270,7 @@ public void testsShouldNotGetCookiesRelatedToCurrentDomainWithoutLeadingPeriod()
   }
 
   @Test
+  @NotWorkingInRemoteBazelBuilds(FIREFOX)
   void testShouldBeAbleToIncludeLeadingPeriodInDomainName() {
     String cookieName = "name";
     assertCookieIsNotPresentWithName(cookieName);
@@ -277,6 +285,7 @@ void testShouldBeAbleToIncludeLeadingPeriodInDomainName() {
 
   @Test
   @NotYetImplemented(SAFARI)
+  @NotWorkingInRemoteBazelBuilds(FIREFOX)
   public void testShouldBeAbleToSetDomainToTheCurrentDomain() throws Exception {
     URI url = new URI(driver.getCurrentUrl());
     String host = url.getHost() + ":" + url.getPort();
@@ -291,6 +300,8 @@ public void testShouldBeAbleToSetDomainToTheCurrentDomain() throws Exception {
 
   @Test
   @NotYetImplemented(SAFARI)
+  @NotWorkingInRemoteBazelBuilds(CHROME)
+  @NotWorkingInRemoteBazelBuilds(FIREFOX)
   public void testShouldWalkThePathToDeleteACookie() {
     Cookie cookie1 = new Cookie.Builder("fish", "cod").build();
     driver.manage().addCookie(cookie1);
@@ -320,6 +331,7 @@ public void testShouldWalkThePathToDeleteACookie() {
 
   @Test
   @NotYetImplemented(SAFARI)
+  @NotWorkingInRemoteBazelBuilds(FIREFOX)
   public void testShouldIgnoreThePortNumberOfTheHostWhenSettingTheCookie() throws Exception {
     URI uri = new URI(driver.getCurrentUrl());
     String host = String.format("%s:%d", uri.getHost(), uri.getPort());
@@ -335,6 +347,8 @@ public void testShouldIgnoreThePortNumberOfTheHostWhenSettingTheCookie() throws
 
   @Test
   @NotYetImplemented(SAFARI)
+  @NotWorkingInRemoteBazelBuilds(CHROME)
+  @NotWorkingInRemoteBazelBuilds(FIREFOX)
   public void testCookieEqualityAfterSetAndGet() {
     driver.get(domainHelper.getUrlForFirstValidHostname("animals"));
 
@@ -379,6 +393,8 @@ public void testRetainsCookieExpiry() {
   @Test
   @Ignore(IE)
   @Ignore(SAFARI)
+  @NotWorkingInRemoteBazelBuilds(CHROME)
+  @NotWorkingInRemoteBazelBuilds(FIREFOX)
   public void canHandleSecureCookie() {
     driver.get(domainHelper.getSecureUrlForFirstValidHostname("animals"));
 
@@ -395,6 +411,8 @@ public void canHandleSecureCookie() {
   @Test
   @Ignore(IE)
   @Ignore(SAFARI)
+  @NotWorkingInRemoteBazelBuilds(CHROME)
+  @NotWorkingInRemoteBazelBuilds(FIREFOX)
   public void testRetainsCookieSecure() {
     driver.get(domainHelper.getSecureUrlForFirstValidHostname("animals"));
 
@@ -411,6 +429,8 @@ public void testRetainsCookieSecure() {
 
   @Test
   @Ignore(SAFARI)
+  @NotWorkingInRemoteBazelBuilds(CHROME)
+  @NotWorkingInRemoteBazelBuilds(FIREFOX)
   public void canHandleHttpOnlyCookie() {
     Cookie addedCookie =
         new Cookie.Builder("fish", "cod").path("/common/animals").isHttpOnly(true).build();
@@ -424,6 +444,8 @@ public void canHandleHttpOnlyCookie() {
 
   @Test
   @Ignore(SAFARI)
+  @NotWorkingInRemoteBazelBuilds(CHROME)
+  @NotWorkingInRemoteBazelBuilds(FIREFOX)
   public void testRetainsHttpOnlyFlag() {
     Cookie addedCookie =
         new Cookie.Builder("fish", "cod").path("/common/animals").isHttpOnly(true).build();

java/test/org/openqa/selenium/JavascriptEnabledDriverTest.java

@@ -23,13 +23,15 @@
 import static org.openqa.selenium.WaitingConditions.elementValueToEqual;
 import static org.openqa.selenium.WaitingConditions.windowToBeSwitchedToWithName;
 import static org.openqa.selenium.support.ui.ExpectedConditions.titleIs;
+import static org.openqa.selenium.testing.drivers.Browser.CHROME;
 import static org.openqa.selenium.testing.drivers.Browser.SAFARI;
 
 import org.junit.jupiter.api.Test;
 import org.openqa.selenium.interactions.Locatable;
 import org.openqa.selenium.support.ui.ExpectedConditions;
 import org.openqa.selenium.testing.JupiterTestBase;
 import org.openqa.selenium.testing.NoDriverAfterTest;
+import org.openqa.selenium.testing.NotWorkingInRemoteBazelBuilds;
 import org.openqa.selenium.testing.NotYetImplemented;
 
 class JavascriptEnabledDriverTest extends JupiterTestBase {
@@ -61,6 +63,7 @@ public void testDocumentShouldReflectLatestDom() {
   }
 
   @Test
+  @NotWorkingInRemoteBazelBuilds(value = CHROME)
   void testShouldWaitForLoadsToCompleteAfterJavascriptCausesANewPageToLoad() {
     driver.get(pages.formPage);
 
@@ -71,6 +74,7 @@ void testShouldWaitForLoadsToCompleteAfterJavascriptCausesANewPageToLoad() {
   }
 
   @Test
+  @NotWorkingInRemoteBazelBuilds(value = CHROME)
   void testShouldBeAbleToFindElementAfterJavascriptCausesANewPageToLoad() {
     driver.get(pages.formPage);
 

java/test/org/openqa/selenium/testing/BUILD.bazel

@@ -22,6 +22,8 @@ java_library(
         "NeedsFreshDriver.java",
         "NoDriverAfterTest.java",
         "NoDriverBeforeTest.java",
+        "NotWorkingInRemoteBazelBuilds.java",
+        "NotWorkingInRemoteBazelBuildsList.java",
         "NotYetImplemented.java",
         "NotYetImplementedList.java",
         "SwitchToTopAfterTest.java",

java/test/org/openqa/selenium/testing/NotWorkingInRemoteBazelBuilds.java

@@ -0,0 +1,18 @@
+package org.openqa.selenium.testing;
+
+import org.openqa.selenium.testing.drivers.Browser;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Repeatable;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+@Repeatable(NotWorkingInRemoteBazelBuildsList.class)
+public @interface NotWorkingInRemoteBazelBuilds {
+
+  Browser value() default Browser.ALL;
+
+}

java/test/org/openqa/selenium/testing/NotWorkingInRemoteBazelBuildsList.java

@@ -0,0 +1,12 @@
+package org.openqa.selenium.testing;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD})
+public @interface NotWorkingInRemoteBazelBuildsList {
+  NotWorkingInRemoteBazelBuilds[] value();
+}

java/test/org/openqa/selenium/testing/SeleniumExtension.java

@@ -67,6 +67,7 @@ public class SeleniumExtension
   private final CaptureLoggingRule captureLoggingRule;
 
   private boolean failedWithNotYetImplemented = false;
+  private boolean failedWithRemoteBuild = false;
 
   public SeleniumExtension() {
     this(Duration.ofSeconds(10), Duration.ofSeconds(5));
@@ -115,6 +116,9 @@ public void beforeEach(ExtensionContext context) throws Exception {
 
     // NotYetImplementedRule
     failedWithNotYetImplemented = false;
+
+    // Remote builds
+    failedWithRemoteBuild = false;
   }
 
   @Override
@@ -144,6 +148,20 @@ public void afterEach(ExtensionContext context) throws Exception {
               testMethod.map(Method::getName).orElse(""),
               current));
     }
+
+    NotWorkingInRemoteBazelBuildsRule notWorkingInRemoteBuilds = new NotWorkingInRemoteBazelBuildsRule(context);
+    boolean isNotExpectedToWork = notWorkingInRemoteBuilds.check();
+
+    if (isNotExpectedToWork && !failedWithRemoteBuild) {
+      Optional<Class<?>> testClass = context.getTestClass();
+      Optional<Method> testMethod = context.getTestMethod();
+      throw new Exception(
+        String.format(
+          "%s.%s is not yet expected to work on remote builds using %s, but it already works!",
+          testClass.map(Class::getName).orElse(""),
+          testMethod.map(Method::getName).orElse(""),
+          Browser.detect()));
+    }
   }
 
   @Override
@@ -199,10 +217,19 @@ public void handleTestExecutionException(ExtensionContext context, Throwable thr
     // NotYetImplementedRule
     NotYetImplementedRule notYetImplementedRule = new NotYetImplementedRule(context);
     if (notYetImplementedRule.check()) {
-      // Expected
       failedWithNotYetImplemented = true;
+    }
+
+    NotWorkingInRemoteBazelBuildsRule notWorkingInRemoteBuilds = new NotWorkingInRemoteBazelBuildsRule(context);
+    if (notWorkingInRemoteBuilds.check()) {
+      failedWithRemoteBuild = true;
+    }
+
+    if (failedWithNotYetImplemented || failedWithRemoteBuild) {
+      // Expected failures.
       return;
     }
+
     throw throwable;
   }
 
@@ -309,6 +336,37 @@ public boolean check() throws Exception {
           findRepeatableAnnotations(element, NotYetImplemented.class);
       return notImplemented(notYetImplementedList) || notImplemented(notYetImplemented.stream());
     }
+
+  }
+
+  private static class NotWorkingInRemoteBazelBuildsRule {
+    ExtensionContext context;
+    private final Browser current = Objects.requireNonNull(Browser.detect());
+
+    public NotWorkingInRemoteBazelBuildsRule(ExtensionContext context) {
+      this.context = context;
+    }
+
+    private boolean notWorkingYet(Optional<NotWorkingInRemoteBazelBuilds> list) {
+      return list.isPresent() && notWorkingYet(Stream.of(list.get()));
+    }
+
+    private boolean notWorkingYet(Stream<NotWorkingInRemoteBazelBuilds> value) {
+      return value.anyMatch(notWorking -> current.matches(notWorking.value()));
+    }
+
+    public boolean check() {
+      if (!Objects.equals("1", System.getenv("REMOTE_BUILD"))) {
+        return false;
+      }
+
+      Optional<AnnotatedElement> element = context.getElement();
+      Optional<NotWorkingInRemoteBazelBuilds> notWorkingList =
+        findAnnotation(element, NotWorkingInRemoteBazelBuilds.class);
+      List<NotWorkingInRemoteBazelBuilds> notWorking =
+        findRepeatableAnnotations(element, NotWorkingInRemoteBazelBuilds.class);
+      return notWorkingYet(notWorkingList) || notWorkingYet(notWorking.stream());
+    }
   }
 
   private static class SwitchToTopRule {