[Improvement][Seatunnel-web] Add support to execute seatunnel-web REST API e2e

.gitignore

@@ -52,3 +52,4 @@ spark-warehouse
 /seatunnel-ui/node/*
 /seatunnel-ui/node_modules
 /seatunnel-ui/node_modules/*
+/seatunnel-web-it/profile/*

pom.xml

@@ -36,6 +36,7 @@
         <module>seatunnel-server</module>
         <module>seatunnel-datasource</module>
         <module>seatunnel-web-dist</module>
+        <module>seatunnel-web-it</module>
     </modules>
 
     <properties>
@@ -68,7 +69,7 @@
         <junit.version>5.9.0</junit.version>
         <commons-collections4.version>4.4</commons-collections4.version>
         <commons-lang3.version>3.4</commons-lang3.version>
-        <guava.version>19.0</guava.version>
+        <guava.version>33.2.1-jre</guava.version>
         <checker.qual.version>3.10.0</checker.qual.version>
         <awaitility.version>4.2.0</awaitility.version>
         <seatunnel-framework.version>2.3.6</seatunnel-framework.version>

seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java

@@ -221,7 +221,10 @@ public Pair<String, String> getTableSyncMaxValue(
     }
 
     private ClassLoader getCustomClassloader(String pluginName) {
-        String getenv = System.getenv(ST_WEB_BASEDIR_PATH);
+        String getenv =
+                System.getenv(ST_WEB_BASEDIR_PATH) == null
+                        ? System.getProperty(ST_WEB_BASEDIR_PATH)
+                        : System.getenv(ST_WEB_BASEDIR_PATH);
         log.info("ST_WEB_BASEDIR_PATH is : " + getenv);
         String libPath = StringUtils.isEmpty(getenv) ? "/datasource" : (getenv + "/datasource");
 

seatunnel-web-it/README.md

@@ -0,0 +1,13 @@
+Build seatunnel-web
+./mvnw clean install -DskipTests
+
+Update mysql database details in src/test/resources/application.yml and Run the seatunnel-web-it integration tests
+./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -DSEATUNNEL_HOME=/some/path/apache-seatunnel-2.3.6 -DST_WEB_BASEDIR_PATH=seatunnel-web-dist/target/apache-seatunnel-web-1.0.0-SNAPSHOT/apache-seatunnel-web-1.0.0-SNAPSHOT
+NOTE: Please remember to update the versions according to the latest supported versions.
+
+If you're using a version of Java higher than Java 8 for running the tests, add the following VM options: 
+-DitJvmArgs="--add-opens java.base/java.lang.invoke=ALL-UNNAMED".
+
+While running integrations tests from IDE, ensure following VM options are set
+SEATUNNEL_HOME=/some/path/apache-seatunnel-2.3.6
+ST_WEB_BASEDIR_PATH=/some/path/seatunnel-web-dist/target/apache-seatunnel-web-1.0.0-SNAPSHOT/apache-seatunnel-web-1.0.0-SNAPSHOT

seatunnel-web-it/pom.xml

@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.seatunnel</groupId>
+        <artifactId>seatunnel-web</artifactId>
+        <version>${revision}</version>
+    </parent>
+
+    <artifactId>seatunnel-web-it</artifactId>
+    <properties>
+        <skipIT>true</skipIT>
+        <itJvmArgs>-Xmx1024m</itJvmArgs>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-engine-server</artifactId>
+            <version>${seatunnel-framework.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.logging.log4j</groupId>
+                    <artifactId>log4j-slf4j-impl</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-server</artifactId>
+            <version>${project.version}</version>
+            <type>pom</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-app</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>${mysql-connector.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-hadoop3-3.1.4-uber</artifactId>
+            <version>${seatunnel-framework.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.avro</groupId>
+                    <artifactId>avro</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-reload4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.logging.log4j</groupId>
+                    <artifactId>log4j-slf4j-impl</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <skip>${skipIT}</skip>
+                    <testFailureIgnore>true</testFailureIgnore>
+                    <forkCount>1</forkCount>
+                    <reuseForks>false</reuseForks>
+                    <argLine>${itJvmArgs}</argLine>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

seatunnel-web-it/src/test/java/org/apache/seatunnel/app/common/SeaTunnelWebCluster.java

@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.common;
+
+import org.apache.seatunnel.app.SeatunnelApplication;
+import org.apache.seatunnel.common.utils.ExceptionUtils;
+import org.apache.seatunnel.engine.common.config.ConfigProvider;
+import org.apache.seatunnel.engine.common.config.SeaTunnelConfig;
+import org.apache.seatunnel.engine.server.SeaTunnelServer;
+import org.apache.seatunnel.engine.server.SeaTunnelServerStarter;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.context.ConfigurableApplicationContext;
+
+import com.hazelcast.config.Config;
+import com.hazelcast.instance.impl.HazelcastInstanceImpl;
+import com.hazelcast.logging.ILogger;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.File;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@Slf4j
+public class SeaTunnelWebCluster {
+    private SeaTunnelServer server;
+    private HazelcastInstanceImpl instance;
+    private ConfigurableApplicationContext applicationContext;
+
+    public void start() {
+        String seatunnelHome = System.getProperty("SEATUNNEL_HOME");
+        if (seatunnelHome == null) {
+            throw new RuntimeException(
+                    "SEATUNNEL_HOME is not set. Please set it before running the tests.");
+        }
+        if (!new File(seatunnelHome).exists()) {
+            throw new RuntimeException(
+                    seatunnelHome
+                            + " does not exist. Please make sure it exists before running the tests");
+        }
+        Config hazelcastConfig = Config.loadDefault();
+        SeaTunnelConfig seaTunnelConfig = ConfigProvider.locateAndGetSeaTunnelConfig();
+        seaTunnelConfig.setHazelcastConfig(hazelcastConfig);
+        instance = SeaTunnelServerStarter.createHazelcastInstance(seaTunnelConfig);
+        server = instance.node.nodeEngine.getService(SeaTunnelServer.SERVICE_NAME);
+        ILogger LOGGER = instance.node.nodeEngine.getLogger(SeaTunnelWebCluster.class);
+
+        // String[] args = {"--spring.profiles.active=h2"};
+        String[] args = {};
+        applicationContext = SpringApplication.run(SeatunnelApplication.class, args);
+        LOGGER.info("SeaTunnel-web server started.");
+        assertTrue(isRunning());
+    }
+
+    public boolean isRunning() {
+        return server.isMasterNode();
+    }
+
+    public void stop() {
+        try {
+            if (applicationContext != null) {
+                int exit = SpringApplication.exit(applicationContext);
+                log.info("Sea tunnel application exited with code: {}", exit);
+            }
+        } catch (Throwable throwable) {
+            log.error("Error stopping application context", throwable);
+        }
+
+        try {
+            if (server != null) {
+                server.shutdown(true);
+            }
+
+            if (instance != null) {
+                instance.shutdown();
+            }
+        } catch (Exception e) {
+            log.error(ExceptionUtils.getMessage(e));
+        }
+    }
+}

seatunnel-web-it/src/test/java/org/apache/seatunnel/app/common/SeatunnelWebTestingBase.java

@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.app.common;
+
+import org.apache.seatunnel.app.domain.request.user.UserLoginReq;
+import org.apache.seatunnel.app.domain.response.user.UserSimpleInfoRes;
+import org.apache.seatunnel.app.utils.JSONTestUtils;
+import org.apache.seatunnel.app.utils.JSONUtils;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.lang.reflect.Field;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+public class SeatunnelWebTestingBase {
+    protected final String baseUrl = "http://localhost:8802/seatunnel/api/v1";
+
+    protected Result<UserSimpleInfoRes> login(UserLoginReq userLoginReq) {
+        String requestBody = JSONUtils.toPrettyJsonString(userLoginReq);
+        String response = sendRequest(url("user/login"), requestBody, "POST");
+        return JSONTestUtils.parseObject(
+                response, new TypeReference<Result<UserSimpleInfoRes>>() {});
+    }
+
+    protected String url(String path) {
+        return String.format("%s/%s?", baseUrl, path);
+    }
+
+    protected String urlWithParam(String pathAndParam) {
+        return String.format("%s/%s", baseUrl, pathAndParam);
+    }
+
+    protected String sendRequest(String url) {
+        return sendRequest(url, null, "GET");
+    }
+
+    protected String sendRequest(String url, String requestBody, String httpMethod) {
+        HttpURLConnection connection = null;
+        try {
+            URL urlObject = new URL(url);
+            connection = (HttpURLConnection) urlObject.openConnection();
+            if ("PATCH".equalsIgnoreCase(httpMethod)) {
+                setRequestMethodUsingReflection(connection, "PATCH");
+            } else {
+                connection.setRequestMethod(httpMethod);
+            }
+
+            connection.setRequestProperty("Content-Type", "application/json");
+            if (!url.endsWith("user/login?")) {
+                connection.setRequestProperty("token", TokenProvider.getToken());
+            }
+            connection.setDoOutput(true);
+            if (requestBody != null) {
+                try (OutputStream os = connection.getOutputStream()) {
+                    byte[] input = requestBody.getBytes("utf-8");
+                    os.write(input, 0, input.length);
+                }
+            }
+            int responseCode = connection.getResponseCode();
+            if (responseCode == HttpURLConnection.HTTP_OK) {
+                return readResponse(connection);
+            } else {
+                String message = "API Request failed with status code: " + responseCode;
+                throw new RuntimeException(message);
+            }
+
+        } catch (Exception e) {
+            throw new RuntimeException(e.getMessage(), e);
+        } finally {
+            if (connection != null) {
+                connection.disconnect();
+            }
+        }
+    }
+
+    private static void setRequestMethodUsingReflection(
+            HttpURLConnection httpURLConnection, String method) throws Exception {
+        try {
+            Field methodField = HttpURLConnection.class.getDeclaredField("method");
+            methodField.setAccessible(true);
+            methodField.set(httpURLConnection, method);
+        } catch (NoSuchFieldException | IllegalAccessException e) {
+            throw new Exception("Failed to set HTTP method to PATCH", e);
+        }
+    }
+
+    private String readResponse(HttpURLConnection connection) throws IOException {
+        BufferedReader rd = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+        String line;
+        StringBuilder result = new StringBuilder();
+        while ((line = rd.readLine()) != null) {
+            result.append(line);
+        }
+        rd.close();
+        return result.toString();
+    }
+}