From a7fdd010c35e594726e666b62c5c87cd9501bd1b Mon Sep 17 00:00:00 2001 From: Simon Mavi Stewart Date: Mon, 5 Jun 2023 21:16:59 +0100 Subject: [PATCH] [bazel] Mark remaining tests as being skipped for remote builds --- .bazelrc | 3 + java/private/selenium_test.bzl | 2 + java/test/org/openqa/selenium/ClickTest.java | 19 +++--- .../selenium/CookieImplementationTest.java | 22 +++++++ .../selenium/JavascriptEnabledDriverTest.java | 4 ++ .../org/openqa/selenium/testing/BUILD.bazel | 2 + .../NotWorkingInRemoteBazelBuilds.java | 18 ++++++ .../NotWorkingInRemoteBazelBuildsList.java | 12 ++++ .../selenium/testing/SeleniumExtension.java | 60 ++++++++++++++++++- 9 files changed, 132 insertions(+), 10 deletions(-) create mode 100644 java/test/org/openqa/selenium/testing/NotWorkingInRemoteBazelBuilds.java create mode 100644 java/test/org/openqa/selenium/testing/NotWorkingInRemoteBazelBuildsList.java diff --git a/.bazelrc b/.bazelrc index 8d03c74078d41..8ab2dbe64f5a8 100644 --- a/.bazelrc +++ b/.bazelrc @@ -136,3 +136,6 @@ build:remote-ci --bes_upload_mode=wait_for_upload_complete # make reasonable usage of everything, everywhere, all at once. build:remote --local_cpu_resources='HOST_CPUS*10' build:remote --local_ram_resources='HOST_RAM*4.0' + +# A small hint that we're running our tests remotely +test:remote --test_env=REMOTE_BUILD=1 diff --git a/java/private/selenium_test.bzl b/java/private/selenium_test.bzl index 6960fd8e4a38d..ba44a9c7f5ce1 100644 --- a/java/private/selenium_test.bzl +++ b/java/private/selenium_test.bzl @@ -77,6 +77,7 @@ def selenium_test(name, test_class, size = "medium", browsers = BROWSERS.keys(), stripped_args.pop("data", None) stripped_args.pop("jvm_flags", None) stripped_args.pop("tags", None) + inherited_env = stripped_args.pop("env_inherit", []) + ["REMOTE_BUILD"] all_tests = [] @@ -94,6 +95,7 @@ def selenium_test(name, test_class, size = "medium", browsers = BROWSERS.keys(), # Only allow linting on the default test tags = BROWSERS[browser]["tags"] + tags + ([] if test == name else ["no-lint"]), data = BROWSERS[browser]["data"] + data, + env_inherit = inherited_env, **stripped_args ) if browser == default_browser: diff --git a/java/test/org/openqa/selenium/ClickTest.java b/java/test/org/openqa/selenium/ClickTest.java index e491a42ebf2d5..72aa253e88166 100644 --- a/java/test/org/openqa/selenium/ClickTest.java +++ b/java/test/org/openqa/selenium/ClickTest.java @@ -17,15 +17,6 @@ package org.openqa.selenium; -import static org.assertj.core.api.Assertions.assertThat; -import static org.openqa.selenium.WaitingConditions.newWindowIsOpened; -import static org.openqa.selenium.WaitingConditions.pageSourceToContain; -import static org.openqa.selenium.support.ui.ExpectedConditions.titleIs; -import static org.openqa.selenium.testing.drivers.Browser.FIREFOX; -import static org.openqa.selenium.testing.drivers.Browser.IE; -import static org.openqa.selenium.testing.drivers.Browser.SAFARI; - -import java.util.Set; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.openqa.selenium.testing.Ignore; @@ -34,6 +25,16 @@ import org.openqa.selenium.testing.NotYetImplemented; import org.openqa.selenium.testing.SwitchToTopAfterTest; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.openqa.selenium.WaitingConditions.newWindowIsOpened; +import static org.openqa.selenium.WaitingConditions.pageSourceToContain; +import static org.openqa.selenium.support.ui.ExpectedConditions.titleIs; +import static org.openqa.selenium.testing.drivers.Browser.FIREFOX; +import static org.openqa.selenium.testing.drivers.Browser.IE; +import static org.openqa.selenium.testing.drivers.Browser.SAFARI; + class ClickTest extends JupiterTestBase { @BeforeEach diff --git a/java/test/org/openqa/selenium/CookieImplementationTest.java b/java/test/org/openqa/selenium/CookieImplementationTest.java index 475bc371b3071..01527171d213a 100644 --- a/java/test/org/openqa/selenium/CookieImplementationTest.java +++ b/java/test/org/openqa/selenium/CookieImplementationTest.java @@ -20,6 +20,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assumptions.assumeTrue; import static org.openqa.selenium.testing.drivers.Browser.ALL; +import static org.openqa.selenium.testing.drivers.Browser.CHROME; +import static org.openqa.selenium.testing.drivers.Browser.FIREFOX; import static org.openqa.selenium.testing.drivers.Browser.IE; import static org.openqa.selenium.testing.drivers.Browser.SAFARI; @@ -32,6 +34,7 @@ import org.openqa.selenium.environment.DomainHelper; import org.openqa.selenium.testing.Ignore; import org.openqa.selenium.testing.JupiterTestBase; +import org.openqa.selenium.testing.NotWorkingInRemoteBazelBuilds; import org.openqa.selenium.testing.NotYetImplemented; import org.openqa.selenium.testing.SwitchToTopAfterTest; @@ -81,6 +84,7 @@ public void testShouldGetCookieByName() { @Test @NotYetImplemented(SAFARI) + @NotWorkingInRemoteBazelBuilds(FIREFOX) public void testShouldBeAbleToAddCookie() { String key = generateUniqueKey(); String value = "foo"; @@ -200,6 +204,8 @@ public void testAddCookiesWithDifferentPathsThatAreRelatedToOurs() { @SwitchToTopAfterTest @Test @Ignore(SAFARI) + @NotWorkingInRemoteBazelBuilds(CHROME) + @NotWorkingInRemoteBazelBuilds(FIREFOX) public void testGetCookiesInAFrame() { driver.get(domainHelper.getUrlForFirstValidHostname("/common/animals")); Cookie cookie1 = new Cookie.Builder("fish", "cod").path("/common/animals").build(); @@ -239,6 +245,7 @@ public void testShouldNotGetCookieOnDifferentDomain() { @Test @NotYetImplemented(SAFARI) + @NotWorkingInRemoteBazelBuilds(FIREFOX) public void testShouldBeAbleToAddToADomainWhichIsRelatedToTheCurrentDomain() { String cookieName = "name"; assertCookieIsNotPresentWithName(cookieName); @@ -263,6 +270,7 @@ public void testsShouldNotGetCookiesRelatedToCurrentDomainWithoutLeadingPeriod() } @Test + @NotWorkingInRemoteBazelBuilds(FIREFOX) void testShouldBeAbleToIncludeLeadingPeriodInDomainName() { String cookieName = "name"; assertCookieIsNotPresentWithName(cookieName); @@ -277,6 +285,7 @@ void testShouldBeAbleToIncludeLeadingPeriodInDomainName() { @Test @NotYetImplemented(SAFARI) + @NotWorkingInRemoteBazelBuilds(FIREFOX) public void testShouldBeAbleToSetDomainToTheCurrentDomain() throws Exception { URI url = new URI(driver.getCurrentUrl()); String host = url.getHost() + ":" + url.getPort(); @@ -291,6 +300,8 @@ public void testShouldBeAbleToSetDomainToTheCurrentDomain() throws Exception { @Test @NotYetImplemented(SAFARI) + @NotWorkingInRemoteBazelBuilds(CHROME) + @NotWorkingInRemoteBazelBuilds(FIREFOX) public void testShouldWalkThePathToDeleteACookie() { Cookie cookie1 = new Cookie.Builder("fish", "cod").build(); driver.manage().addCookie(cookie1); @@ -320,6 +331,7 @@ public void testShouldWalkThePathToDeleteACookie() { @Test @NotYetImplemented(SAFARI) + @NotWorkingInRemoteBazelBuilds(FIREFOX) public void testShouldIgnoreThePortNumberOfTheHostWhenSettingTheCookie() throws Exception { URI uri = new URI(driver.getCurrentUrl()); String host = String.format("%s:%d", uri.getHost(), uri.getPort()); @@ -335,6 +347,8 @@ public void testShouldIgnoreThePortNumberOfTheHostWhenSettingTheCookie() throws @Test @NotYetImplemented(SAFARI) + @NotWorkingInRemoteBazelBuilds(CHROME) + @NotWorkingInRemoteBazelBuilds(FIREFOX) public void testCookieEqualityAfterSetAndGet() { driver.get(domainHelper.getUrlForFirstValidHostname("animals")); @@ -379,6 +393,8 @@ public void testRetainsCookieExpiry() { @Test @Ignore(IE) @Ignore(SAFARI) + @NotWorkingInRemoteBazelBuilds(CHROME) + @NotWorkingInRemoteBazelBuilds(FIREFOX) public void canHandleSecureCookie() { driver.get(domainHelper.getSecureUrlForFirstValidHostname("animals")); @@ -395,6 +411,8 @@ public void canHandleSecureCookie() { @Test @Ignore(IE) @Ignore(SAFARI) + @NotWorkingInRemoteBazelBuilds(CHROME) + @NotWorkingInRemoteBazelBuilds(FIREFOX) public void testRetainsCookieSecure() { driver.get(domainHelper.getSecureUrlForFirstValidHostname("animals")); @@ -411,6 +429,8 @@ public void testRetainsCookieSecure() { @Test @Ignore(SAFARI) + @NotWorkingInRemoteBazelBuilds(CHROME) + @NotWorkingInRemoteBazelBuilds(FIREFOX) public void canHandleHttpOnlyCookie() { Cookie addedCookie = new Cookie.Builder("fish", "cod").path("/common/animals").isHttpOnly(true).build(); @@ -424,6 +444,8 @@ public void canHandleHttpOnlyCookie() { @Test @Ignore(SAFARI) + @NotWorkingInRemoteBazelBuilds(CHROME) + @NotWorkingInRemoteBazelBuilds(FIREFOX) public void testRetainsHttpOnlyFlag() { Cookie addedCookie = new Cookie.Builder("fish", "cod").path("/common/animals").isHttpOnly(true).build(); diff --git a/java/test/org/openqa/selenium/JavascriptEnabledDriverTest.java b/java/test/org/openqa/selenium/JavascriptEnabledDriverTest.java index 5c6d1b722da59..91587cc17e497 100644 --- a/java/test/org/openqa/selenium/JavascriptEnabledDriverTest.java +++ b/java/test/org/openqa/selenium/JavascriptEnabledDriverTest.java @@ -23,6 +23,7 @@ import static org.openqa.selenium.WaitingConditions.elementValueToEqual; import static org.openqa.selenium.WaitingConditions.windowToBeSwitchedToWithName; import static org.openqa.selenium.support.ui.ExpectedConditions.titleIs; +import static org.openqa.selenium.testing.drivers.Browser.CHROME; import static org.openqa.selenium.testing.drivers.Browser.SAFARI; import org.junit.jupiter.api.Test; @@ -30,6 +31,7 @@ import org.openqa.selenium.support.ui.ExpectedConditions; import org.openqa.selenium.testing.JupiterTestBase; import org.openqa.selenium.testing.NoDriverAfterTest; +import org.openqa.selenium.testing.NotWorkingInRemoteBazelBuilds; import org.openqa.selenium.testing.NotYetImplemented; class JavascriptEnabledDriverTest extends JupiterTestBase { @@ -61,6 +63,7 @@ public void testDocumentShouldReflectLatestDom() { } @Test + @NotWorkingInRemoteBazelBuilds(value = CHROME) void testShouldWaitForLoadsToCompleteAfterJavascriptCausesANewPageToLoad() { driver.get(pages.formPage); @@ -71,6 +74,7 @@ void testShouldWaitForLoadsToCompleteAfterJavascriptCausesANewPageToLoad() { } @Test + @NotWorkingInRemoteBazelBuilds(value = CHROME) void testShouldBeAbleToFindElementAfterJavascriptCausesANewPageToLoad() { driver.get(pages.formPage); diff --git a/java/test/org/openqa/selenium/testing/BUILD.bazel b/java/test/org/openqa/selenium/testing/BUILD.bazel index acbc99b90a587..04b6a55f62a53 100644 --- a/java/test/org/openqa/selenium/testing/BUILD.bazel +++ b/java/test/org/openqa/selenium/testing/BUILD.bazel @@ -22,6 +22,8 @@ java_library( "NeedsFreshDriver.java", "NoDriverAfterTest.java", "NoDriverBeforeTest.java", + "NotWorkingInRemoteBazelBuilds.java", + "NotWorkingInRemoteBazelBuildsList.java", "NotYetImplemented.java", "NotYetImplementedList.java", "SwitchToTopAfterTest.java", diff --git a/java/test/org/openqa/selenium/testing/NotWorkingInRemoteBazelBuilds.java b/java/test/org/openqa/selenium/testing/NotWorkingInRemoteBazelBuilds.java new file mode 100644 index 0000000000000..c354fd6618d38 --- /dev/null +++ b/java/test/org/openqa/selenium/testing/NotWorkingInRemoteBazelBuilds.java @@ -0,0 +1,18 @@ +package org.openqa.selenium.testing; + +import org.openqa.selenium.testing.drivers.Browser; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Repeatable; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +@Repeatable(NotWorkingInRemoteBazelBuildsList.class) +public @interface NotWorkingInRemoteBazelBuilds { + + Browser value() default Browser.ALL; + +} diff --git a/java/test/org/openqa/selenium/testing/NotWorkingInRemoteBazelBuildsList.java b/java/test/org/openqa/selenium/testing/NotWorkingInRemoteBazelBuildsList.java new file mode 100644 index 0000000000000..2e50c149e3545 --- /dev/null +++ b/java/test/org/openqa/selenium/testing/NotWorkingInRemoteBazelBuildsList.java @@ -0,0 +1,12 @@ +package org.openqa.selenium.testing; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD}) +public @interface NotWorkingInRemoteBazelBuildsList { + NotWorkingInRemoteBazelBuilds[] value(); +} diff --git a/java/test/org/openqa/selenium/testing/SeleniumExtension.java b/java/test/org/openqa/selenium/testing/SeleniumExtension.java index 3ad9e1421b74b..eb849d3324948 100644 --- a/java/test/org/openqa/selenium/testing/SeleniumExtension.java +++ b/java/test/org/openqa/selenium/testing/SeleniumExtension.java @@ -67,6 +67,7 @@ public class SeleniumExtension private final CaptureLoggingRule captureLoggingRule; private boolean failedWithNotYetImplemented = false; + private boolean failedWithRemoteBuild = false; public SeleniumExtension() { this(Duration.ofSeconds(10), Duration.ofSeconds(5)); @@ -115,6 +116,9 @@ public void beforeEach(ExtensionContext context) throws Exception { // NotYetImplementedRule failedWithNotYetImplemented = false; + + // Remote builds + failedWithRemoteBuild = false; } @Override @@ -144,6 +148,20 @@ public void afterEach(ExtensionContext context) throws Exception { testMethod.map(Method::getName).orElse(""), current)); } + + NotWorkingInRemoteBazelBuildsRule notWorkingInRemoteBuilds = new NotWorkingInRemoteBazelBuildsRule(context); + boolean isNotExpectedToWork = notWorkingInRemoteBuilds.check(); + + if (isNotExpectedToWork && !failedWithRemoteBuild) { + Optional> testClass = context.getTestClass(); + Optional testMethod = context.getTestMethod(); + throw new Exception( + String.format( + "%s.%s is not yet expected to work on remote builds using %s, but it already works!", + testClass.map(Class::getName).orElse(""), + testMethod.map(Method::getName).orElse(""), + Browser.detect())); + } } @Override @@ -199,10 +217,19 @@ public void handleTestExecutionException(ExtensionContext context, Throwable thr // NotYetImplementedRule NotYetImplementedRule notYetImplementedRule = new NotYetImplementedRule(context); if (notYetImplementedRule.check()) { - // Expected failedWithNotYetImplemented = true; + } + + NotWorkingInRemoteBazelBuildsRule notWorkingInRemoteBuilds = new NotWorkingInRemoteBazelBuildsRule(context); + if (notWorkingInRemoteBuilds.check()) { + failedWithRemoteBuild = true; + } + + if (failedWithNotYetImplemented || failedWithRemoteBuild) { + // Expected failures. return; } + throw throwable; } @@ -309,6 +336,37 @@ public boolean check() throws Exception { findRepeatableAnnotations(element, NotYetImplemented.class); return notImplemented(notYetImplementedList) || notImplemented(notYetImplemented.stream()); } + + } + + private static class NotWorkingInRemoteBazelBuildsRule { + ExtensionContext context; + private final Browser current = Objects.requireNonNull(Browser.detect()); + + public NotWorkingInRemoteBazelBuildsRule(ExtensionContext context) { + this.context = context; + } + + private boolean notWorkingYet(Optional list) { + return list.isPresent() && notWorkingYet(Stream.of(list.get())); + } + + private boolean notWorkingYet(Stream value) { + return value.anyMatch(notWorking -> current.matches(notWorking.value())); + } + + public boolean check() { + if (!Objects.equals("1", System.getenv("REMOTE_BUILD"))) { + return false; + } + + Optional element = context.getElement(); + Optional notWorkingList = + findAnnotation(element, NotWorkingInRemoteBazelBuilds.class); + List notWorking = + findRepeatableAnnotations(element, NotWorkingInRemoteBazelBuilds.class); + return notWorkingYet(notWorkingList) || notWorkingYet(notWorking.stream()); + } } private static class SwitchToTopRule {