From 3505d455f4b44c9a7560f7ba2768b15cd29fbcd8 Mon Sep 17 00:00:00 2001 From: Todd Baert Date: Thu, 3 Aug 2023 10:54:37 -0400 Subject: [PATCH] chore: flagd e2e tests (#387) Signed-off-by: Todd Baert --- .github/workflows/ci.yml | 9 +- .gitmodules | 3 + pom.xml | 19 ++ providers/flagd/CONTRIBUTING.md | 27 ++ providers/flagd/pom.xml | 64 +++- .../providers/flagd/e2e/RunCucumberTest.java | 16 + .../providers/flagd/e2e/StepDefinitions.java | 292 ++++++++++++++++++ .../src/test/resources/features/.gitignore | 1 + .../src/test/resources/features/.gitkeep | 0 providers/flagd/test-harness | 1 + 10 files changed, 430 insertions(+), 2 deletions(-) create mode 100644 providers/flagd/CONTRIBUTING.md create mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunCucumberTest.java create mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/StepDefinitions.java create mode 100644 providers/flagd/src/test/resources/features/.gitignore create mode 100644 providers/flagd/src/test/resources/features/.gitkeep create mode 160000 providers/flagd/test-harness diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc581c6d7..34b1a327c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,13 @@ on: jobs: main: runs-on: ubuntu-latest + services: + # flagd-testbed for flagd-provider e2e tests + flagd: + image: ghcr.io/open-feature/flagd-testbed:latest + ports: + - 8013:8013 + steps: - name: Checkout Repository uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 @@ -31,4 +38,4 @@ jobs: ${{ runner.os }}-maven- - name: Maven Verify - run: mvn --batch-mode clean verify + run: mvn --batch-mode --activate-profiles e2e clean verify diff --git a/.gitmodules b/.gitmodules index 492e41f11..c1576cf27 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "providers/flagd/schemas"] path = providers/flagd/schemas url = https://github.com/open-feature/schemas.git +[submodule "providers/flagd/test-harness"] + path = providers/flagd/test-harness + url = https://github.com/open-feature/test-harness.git diff --git a/pom.xml b/pom.xml index 80b03b3cb..b88b11911 100644 --- a/pom.xml +++ b/pom.xml @@ -47,6 +47,7 @@ 5.10.0 UTF-8 UTF-8 + @@ -161,6 +162,20 @@ 2.0.2 test + + + io.cucumber + cucumber-java + 7.13.0 + test + + + + io.cucumber + cucumber-junit-platform-engine + 7.13.0 + test + @@ -176,6 +191,10 @@ maven-surefire-plugin 3.1.2 + + + ${testExclusions} + ${surefireArgLine} diff --git a/providers/flagd/CONTRIBUTING.md b/providers/flagd/CONTRIBUTING.md new file mode 100644 index 000000000..7622e770f --- /dev/null +++ b/providers/flagd/CONTRIBUTING.md @@ -0,0 +1,27 @@ +# flagd Provider for OpenFeature + +## Building + +`mvn compile` will pull the `schemas` submodule, and build the gRPC/protobug resources. +Note that in some editors, you will need to disable some automatic compilation options to prevent your editor from cleaning them. +In vscode for instance, the following settings are recommended: + +```json +{ + "java.configuration.updateBuildConfiguration": "automatic", + "java.autobuild.enabled": false, + "java.compile.nullAnalysis.mode": "automatic" +} +``` + +## End-to-End Tests + +The continuous integration runs a set of [gherkin e2e tests](https://github.com/open-feature/test-harness/blob/main/features/evaluation.feature) using [`flagd`](https://github.com/open-feature/flagd). These tests do not run with the default maven profile. If you'd like to run them locally, you can start the flagd testbed with + +``` +docker run -p 8013:8013 ghcr.io/open-feature/flagd-testbed:latest +``` +and then run +``` +mvn test -P e2e +``` \ No newline at end of file diff --git a/providers/flagd/pom.xml b/providers/flagd/pom.xml index 2eaf4fad5..d521f5137 100644 --- a/providers/flagd/pom.xml +++ b/providers/flagd/pom.xml @@ -12,6 +12,12 @@ flagd 0.6.0 + + + **/e2e/*.java + + + flagd FlagD provider for Java https://openfeature.dev @@ -115,7 +121,7 @@ submodule update --init - --recursive + schemas @@ -158,4 +164,60 @@ + + + + e2e + + + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + update-test-harness-submodule + validate + + exec + + + + git + + submodule + update + --init + test-harness + + + + + copy-gherkin-tests + validate + + exec + + + + + cp + + test-harness/features/evaluation.feature + src/test/resources/features/ + + + + + + + + + + \ No newline at end of file diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunCucumberTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunCucumberTest.java new file mode 100644 index 000000000..a185d3c6e --- /dev/null +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunCucumberTest.java @@ -0,0 +1,16 @@ +package dev.openfeature.contrib.providers.flagd.e2e; + +import org.junit.platform.suite.api.ConfigurationParameter; +import org.junit.platform.suite.api.IncludeEngines; +import org.junit.platform.suite.api.SelectClasspathResource; +import org.junit.platform.suite.api.Suite; + +import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME; + +@Suite +@IncludeEngines("cucumber") +@SelectClasspathResource("features") +@ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "pretty") +public class RunCucumberTest { + +} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/StepDefinitions.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/StepDefinitions.java new file mode 100644 index 000000000..fcc5f15cd --- /dev/null +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/StepDefinitions.java @@ -0,0 +1,292 @@ +package dev.openfeature.contrib.providers.flagd.e2e; + +import dev.openfeature.contrib.providers.flagd.FlagdProvider; +import dev.openfeature.sdk.Client; +import dev.openfeature.sdk.EvaluationContext; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.ImmutableContext; +import dev.openfeature.sdk.OpenFeatureAPI; +import dev.openfeature.sdk.Reason; +import dev.openfeature.sdk.Structure; +import dev.openfeature.sdk.Value; +import io.cucumber.java.BeforeAll; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class StepDefinitions { + + private static Client client; + private boolean booleanFlagValue; + private String stringFlagValue; + private int intFlagValue; + private double doubleFlagValue; + private Value objectFlagValue; + + private FlagEvaluationDetails booleanFlagDetails; + private FlagEvaluationDetails stringFlagDetails; + private FlagEvaluationDetails intFlagDetails; + private FlagEvaluationDetails doubleFlagDetails; + private FlagEvaluationDetails objectFlagDetails; + + private String contextAwareFlagKey; + private String contextAwareDefaultValue; + private EvaluationContext context; + private String contextAwareValue; + + private String notFoundFlagKey; + private String notFoundDefaultValue; + private FlagEvaluationDetails notFoundDetails; + private String typeErrorFlagKey; + private int typeErrorDefaultValue; + private FlagEvaluationDetails typeErrorDetails; + + @BeforeAll() + @Given("a provider is registered with cache disabled") + public static void setup() { + // TODO: when the FlagdProvider is updated to support caching, we might need to disable it here for this test to work as expected. + FlagdProvider provider = new FlagdProvider(); + provider.setDeadline(3000); // set a generous deadline, to prevent timeouts in actions + OpenFeatureAPI.getInstance().setProvider(provider); + client = OpenFeatureAPI.getInstance().getClient(); + } + + /* + * Basic evaluation + */ + + // boolean value + @When("a boolean flag with key {string} is evaluated with default value {string}") + public void a_boolean_flag_with_key_boolean_flag_is_evaluated_with_default_value_false(String flagKey, + String defaultValue) { + this.booleanFlagValue = client.getBooleanValue(flagKey, Boolean.valueOf(defaultValue)); + } + + @Then("the resolved boolean value should be {string}") + public void the_resolved_boolean_value_should_be_true(String expected) { + assertEquals(Boolean.valueOf(expected), this.booleanFlagValue); + } + + // string value + @When("a string flag with key {string} is evaluated with default value {string}") + public void a_string_flag_with_key_is_evaluated_with_default_value(String flagKey, String defaultValue) { + this.stringFlagValue = client.getStringValue(flagKey, defaultValue); + } + + @Then("the resolved string value should be {string}") + public void the_resolved_string_value_should_be(String expected) { + assertEquals(expected, this.stringFlagValue); + } + + // integer value + @When("an integer flag with key {string} is evaluated with default value {int}") + public void an_integer_flag_with_key_is_evaluated_with_default_value(String flagKey, Integer defaultValue) { + this.intFlagValue = client.getIntegerValue(flagKey, defaultValue); + } + + @Then("the resolved integer value should be {int}") + public void the_resolved_integer_value_should_be(int expected) { + assertEquals(expected, this.intFlagValue); + } + + // float/double value + @When("a float flag with key {string} is evaluated with default value {double}") + public void a_float_flag_with_key_is_evaluated_with_default_value(String flagKey, double defaultValue) { + this.doubleFlagValue = client.getDoubleValue(flagKey, defaultValue); + } + + @Then("the resolved float value should be {double}") + public void the_resolved_float_value_should_be(double expected) { + assertEquals(expected, this.doubleFlagValue); + } + + // object value + @When("an object flag with key {string} is evaluated with a null default value") + public void an_object_flag_with_key_is_evaluated_with_a_null_default_value(String flagKey) { + this.objectFlagValue = client.getObjectValue(flagKey, new Value()); + } + + @Then("the resolved object value should be contain fields {string}, {string}, and {string}, with values {string}, {string} and {int}, respectively") + public void the_resolved_object_value_should_be_contain_fields_and_with_values_and_respectively(String boolField, + String stringField, String numberField, String boolValue, String stringValue, int numberValue) { + Structure structure = this.objectFlagValue.asStructure(); + + assertEquals(Boolean.valueOf(boolValue), structure.asMap().get(boolField).asBoolean()); + assertEquals(stringValue, structure.asMap().get(stringField).asString()); + assertEquals(numberValue, structure.asMap().get(numberField).asInteger()); + } + + /* + * Detailed evaluation + */ + + // boolean details + @When("a boolean flag with key {string} is evaluated with details and default value {string}") + public void a_boolean_flag_with_key_is_evaluated_with_details_and_default_value(String flagKey, + String defaultValue) { + this.booleanFlagDetails = client.getBooleanDetails(flagKey, Boolean.valueOf(defaultValue)); + } + + @Then("the resolved boolean details value should be {string}, the variant should be {string}, and the reason should be {string}") + public void the_resolved_boolean_value_should_be_the_variant_should_be_and_the_reason_should_be( + String expectedValue, + String expectedVariant, String expectedReason) { + assertEquals(Boolean.valueOf(expectedValue), booleanFlagDetails.getValue()); + assertEquals(expectedVariant, booleanFlagDetails.getVariant()); + assertEquals(expectedReason, booleanFlagDetails.getReason()); + } + + // string details + @When("a string flag with key {string} is evaluated with details and default value {string}") + public void a_string_flag_with_key_is_evaluated_with_details_and_default_value(String flagKey, + String defaultValue) { + this.stringFlagDetails = client.getStringDetails(flagKey, defaultValue); + } + + @Then("the resolved string details value should be {string}, the variant should be {string}, and the reason should be {string}") + public void the_resolved_string_value_should_be_the_variant_should_be_and_the_reason_should_be(String expectedValue, + String expectedVariant, String expectedReason) { + assertEquals(expectedValue, this.stringFlagDetails.getValue()); + assertEquals(expectedVariant, this.stringFlagDetails.getVariant()); + assertEquals(expectedReason, this.stringFlagDetails.getReason()); + } + + // integer details + @When("an integer flag with key {string} is evaluated with details and default value {int}") + public void an_integer_flag_with_key_is_evaluated_with_details_and_default_value(String flagKey, int defaultValue) { + this.intFlagDetails = client.getIntegerDetails(flagKey, defaultValue); + } + + @Then("the resolved integer details value should be {int}, the variant should be {string}, and the reason should be {string}") + public void the_resolved_integer_value_should_be_the_variant_should_be_and_the_reason_should_be(int expectedValue, + String expectedVariant, String expectedReason) { + assertEquals(expectedValue, this.intFlagDetails.getValue()); + assertEquals(expectedVariant, this.intFlagDetails.getVariant()); + assertEquals(expectedReason, this.intFlagDetails.getReason()); + } + + // float/double details + @When("a float flag with key {string} is evaluated with details and default value {double}") + public void a_float_flag_with_key_is_evaluated_with_details_and_default_value(String flagKey, double defaultValue) { + this.doubleFlagDetails = client.getDoubleDetails(flagKey, defaultValue); + } + + @Then("the resolved float details value should be {double}, the variant should be {string}, and the reason should be {string}") + public void the_resolved_float_value_should_be_the_variant_should_be_and_the_reason_should_be(double expectedValue, + String expectedVariant, String expectedReason) { + assertEquals(expectedValue, this.doubleFlagDetails.getValue()); + assertEquals(expectedVariant, this.doubleFlagDetails.getVariant()); + assertEquals(expectedReason, this.doubleFlagDetails.getReason()); + } + + // object details + @When("an object flag with key {string} is evaluated with details and a null default value") + public void an_object_flag_with_key_is_evaluated_with_details_and_a_null_default_value(String flagKey) { + this.objectFlagDetails = client.getObjectDetails(flagKey, new Value()); + } + + @Then("the resolved object details value should be contain fields {string}, {string}, and {string}, with values {string}, {string} and {int}, respectively") + public void the_resolved_object_value_should_be_contain_fields_and_with_values_and_respectively_again( + String boolField, + String stringField, String numberField, String boolValue, String stringValue, int numberValue) { + Structure structure = this.objectFlagDetails.getValue().asStructure(); + + assertEquals(Boolean.valueOf(boolValue), structure.asMap().get(boolField).asBoolean()); + assertEquals(stringValue, structure.asMap().get(stringField).asString()); + assertEquals(numberValue, structure.asMap().get(numberField).asInteger()); + } + + @Then("the variant should be {string}, and the reason should be {string}") + public void the_variant_should_be_and_the_reason_should_be(String expectedVariant, String expectedReason) { + assertEquals(expectedVariant, this.objectFlagDetails.getVariant()); + assertEquals(expectedReason, this.objectFlagDetails.getReason()); + } + + /* + * Context-aware evaluation + */ + + @When("context contains keys {string}, {string}, {string}, {string} with values {string}, {string}, {int}, {string}") + public void context_contains_keys_with_values(String field1, String field2, String field3, String field4, + String value1, String value2, Integer value3, String value4) { + Map attributes = new HashMap<>(); + attributes.put(field1, new Value(value1)); + attributes.put(field2, new Value(value2)); + attributes.put(field3, new Value(value3)); + attributes.put(field4, new Value(Boolean.valueOf(value4))); + this.context = new ImmutableContext(attributes); + } + + @When("a flag with key {string} is evaluated with default value {string}") + public void an_a_flag_with_key_is_evaluated(String flagKey, String defaultValue) { + contextAwareFlagKey = flagKey; + contextAwareDefaultValue = defaultValue; + contextAwareValue = client.getStringValue(flagKey, contextAwareDefaultValue, context); + + } + + @Then("the resolved string response should be {string}") + public void the_resolved_string_response_should_be(String expected) { + assertEquals(expected, this.contextAwareValue); + } + + @Then("the resolved flag value is {string} when the context is empty") + public void the_resolved_flag_value_is_when_the_context_is_empty(String expected) { + String emptyContextValue = client.getStringValue(contextAwareFlagKey, contextAwareDefaultValue, + new ImmutableContext()); + assertEquals(expected, emptyContextValue); + } + + /* + * Errors + */ + + // not found + @When("a non-existent string flag with key {string} is evaluated with details and a default value {string}") + public void a_non_existent_string_flag_with_key_is_evaluated_with_details_and_a_default_value(String flagKey, + String defaultValue) { + notFoundFlagKey = flagKey; + notFoundDefaultValue = defaultValue; + notFoundDetails = client.getStringDetails(notFoundFlagKey, notFoundDefaultValue); + } + + @Then("the default string value should be returned") + public void then_the_default_string_value_should_be_returned() { + assertEquals(notFoundDefaultValue, notFoundDetails.getValue()); + } + + @Then("the reason should indicate an error and the error code should indicate a missing flag with {string}") + public void the_reason_should_indicate_an_error_and_the_error_code_should_be_flag_not_found(String errorCode) { + assertEquals(Reason.ERROR.toString(), notFoundDetails.getReason()); + assertTrue(notFoundDetails.getErrorMessage().contains(errorCode)); + // TODO: add errorCode assertion once flagd provider is updated. + } + + // type mismatch + @When("a string flag with key {string} is evaluated as an integer, with details and a default value {int}") + public void a_string_flag_with_key_is_evaluated_as_an_integer_with_details_and_a_default_value(String flagKey, + int defaultValue) { + typeErrorFlagKey = flagKey; + typeErrorDefaultValue = defaultValue; + typeErrorDetails = client.getIntegerDetails(typeErrorFlagKey, typeErrorDefaultValue); + } + + @Then("the default integer value should be returned") + public void then_the_default_integer_value_should_be_returned() { + assertEquals(typeErrorDefaultValue, typeErrorDetails.getValue()); + } + + @Then("the reason should indicate an error and the error code should indicate a type mismatch with {string}") + public void the_reason_should_indicate_an_error_and_the_error_code_should_be_type_mismatch(String errorCode) { + assertEquals(Reason.ERROR.toString(), typeErrorDetails.getReason()); + assertTrue(typeErrorDetails.getErrorMessage().contains(errorCode)); + // TODO: add errorCode assertion once flagd provider is updated. + } + +} diff --git a/providers/flagd/src/test/resources/features/.gitignore b/providers/flagd/src/test/resources/features/.gitignore new file mode 100644 index 000000000..d2092accf --- /dev/null +++ b/providers/flagd/src/test/resources/features/.gitignore @@ -0,0 +1 @@ +*.feature \ No newline at end of file diff --git a/providers/flagd/src/test/resources/features/.gitkeep b/providers/flagd/src/test/resources/features/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/providers/flagd/test-harness b/providers/flagd/test-harness new file mode 160000 index 000000000..bd13458f7 --- /dev/null +++ b/providers/flagd/test-harness @@ -0,0 +1 @@ +Subproject commit bd13458f7e3587ab2ed98b8017bea3c2eb472cc9