chore: flagd e2e tests (#387)

Signed-off-by: Todd Baert <todd.baert@dynatrace.com>

.github/workflows/ci.yml

@@ -11,6 +11,13 @@ on:
 jobs:
   main:
     runs-on: ubuntu-latest
+    services:
+      # flagd-testbed for flagd-provider e2e tests
+      flagd:
+        image: ghcr.io/open-feature/flagd-testbed:latest
+        ports:
+          - 8013:8013
+
     steps:
       - name: Checkout Repository
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3
@@ -31,4 +38,4 @@ jobs:
             ${{ runner.os }}-maven-
 
       - name: Maven Verify
-        run: mvn --batch-mode clean verify
+        run: mvn --batch-mode --activate-profiles e2e clean verify

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "providers/flagd/schemas"]
 	path = providers/flagd/schemas
 	url = https://github.com/open-feature/schemas.git
+[submodule "providers/flagd/test-harness"]
+	path = providers/flagd/test-harness
+	url = https://github.com/open-feature/test-harness.git

pom.xml

@@ -47,6 +47,7 @@
         <junit.jupiter.version>5.10.0</junit.jupiter.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+
     </properties>
 
     <dependencies>
@@ -161,6 +162,20 @@
             <version>2.0.2</version>
             <scope>test</scope>
           </dependency>
+
+          <dependency>
+            <groupId>io.cucumber</groupId>
+            <artifactId>cucumber-java</artifactId>
+            <version>7.13.0</version>
+            <scope>test</scope>
+          </dependency>
+
+          <dependency>
+            <groupId>io.cucumber</groupId>
+            <artifactId>cucumber-junit-platform-engine</artifactId>
+            <version>7.13.0</version>
+            <scope>test</scope>
+          </dependency>
         <!-- end test -->
     </dependencies>
 
@@ -176,6 +191,10 @@
                 <artifactId>maven-surefire-plugin</artifactId>
                 <version>3.1.2</version>
                 <configuration>
+                    <excludes>
+                        <!-- tests to exclude -->
+                        <exclude>${testExclusions}</exclude>
+                    </excludes>
                     <argLine>${surefireArgLine}</argLine>
                 </configuration>
             </plugin>

providers/flagd/CONTRIBUTING.md

@@ -0,0 +1,27 @@
+# flagd Provider for OpenFeature
+
+## Building
+
+`mvn compile` will pull the `schemas` submodule, and build the gRPC/protobug resources.
+Note that in some editors, you will need to disable some automatic compilation options to prevent your editor from cleaning them.
+In vscode for instance, the following settings are recommended:
+
+```json
+{
+  "java.configuration.updateBuildConfiguration": "automatic",
+  "java.autobuild.enabled": false,
+  "java.compile.nullAnalysis.mode": "automatic"
+}
+```
+
+## End-to-End Tests
+
+The continuous integration runs a set of [gherkin e2e tests](https://github.com/open-feature/test-harness/blob/main/features/evaluation.feature) using [`flagd`](https://github.com/open-feature/flagd). These tests do not run with the default maven profile. If you'd like to run them locally, you can start the flagd testbed with
+
+```
+docker run -p 8013:8013 ghcr.io/open-feature/flagd-testbed:latest
+```
+and then run 
+```
+mvn test -P e2e
+```

providers/flagd/pom.xml

@@ -12,6 +12,12 @@
     <artifactId>flagd</artifactId>
     <version>0.6.0</version> <!--x-release-please-version -->
 
+    <properties>
+        <!-- exclusion expression for e2e tests -->
+        <testExclusions>**/e2e/*.java</testExclusions>
+    </properties>
+
+
     <name>flagd</name>
     <description>FlagD provider for Java</description>
     <url>https://openfeature.dev</url>
@@ -115,7 +121,7 @@
                                 <argument>submodule</argument>
                                 <argument>update</argument>
                                 <argument>--init</argument>
-                                <argument>--recursive</argument>
+                                <argument>schemas</argument>
                             </arguments>
                         </configuration>
                     </execution>
@@ -158,4 +164,60 @@
         </plugins>
     </build>
 
+    <profiles>
+        <profile>
+        <!-- this profile handles running the flagd e2e tests -->
+        <id>e2e</id>
+        <properties>
+            <!-- run the e2e tests by clearing the exclusions -->
+            <testExclusions/>
+        </properties>  
+        <build>
+            <plugins>
+            <!-- pull the gherkin tests as a git submodule  -->
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <version>3.1.0</version>
+                <executions>
+                <execution>
+                    <id>update-test-harness-submodule</id>
+                    <phase>validate</phase>
+                    <goals>
+                    <goal>exec</goal>
+                    </goals>
+                    <configuration>
+                    <!-- run: git submodule update \-\-init \-\-recursive -->
+                    <executable>git</executable>
+                    <arguments>
+                        <argument>submodule</argument>
+                        <argument>update</argument>
+                        <argument>--init</argument>
+                        <argument>test-harness</argument>
+                    </arguments>
+                    </configuration>
+                </execution>
+                <execution>
+                    <id>copy-gherkin-tests</id>
+                    <phase>validate</phase>
+                    <goals>
+                    <goal>exec</goal>
+                    </goals>
+                    <configuration>
+                    <!-- copy the feature spec we want to test into resources so them can be easily loaded -->
+                    <!-- run: cp test-harness/features/evaluation.feature src/test/resources/features/ -->
+                    <executable>cp</executable>
+                    <arguments>
+                        <argument>test-harness/features/evaluation.feature</argument>
+                        <argument>src/test/resources/features/</argument>
+                    </arguments>
+                    </configuration>
+                </execution>
+                </executions>
+            </plugin>
+            </plugins>
+        </build>
+        </profile>
+    </profiles>
+
 </project>

providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunCucumberTest.java

@@ -0,0 +1,16 @@
+package dev.openfeature.contrib.providers.flagd.e2e;
+
+import org.junit.platform.suite.api.ConfigurationParameter;
+import org.junit.platform.suite.api.IncludeEngines;
+import org.junit.platform.suite.api.SelectClasspathResource;
+import org.junit.platform.suite.api.Suite;
+
+import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME;
+
+@Suite
+@IncludeEngines("cucumber")
+@SelectClasspathResource("features")
+@ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "pretty")
+public class RunCucumberTest {
+
+}