revert: Datasource need test binding to create prod (#14381)

We should keep our step as less as possible, for now, we have to
test datasource and binding it to the prod datasource, I think it will
make our users do not thing to use datasource, so I do like to revert
it and keep step as less as possible

related to #11670

(cherry picked from commit 221df4a)

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java

@@ -82,7 +82,7 @@ public class DataSourceController extends BaseController {
      *
      * @param loginUser login user
      * @param jsonStr   datasource param
-     *                  example: {"type":"MYSQL","name":"txx","note":"","host":"localhost","port":3306,"principal":"","javaSecurityKrb5Conf":"","loginUserKeytabUsername":"","loginUserKeytabPath":"","userName":"root","password":"xxx","database":"ds","connectType":"","other":{"serverTimezone":"GMT-8"},"id":2,"testFlag":0,"bindTestId":1}
+     *                  example: {"type":"MYSQL","name":"txx","note":"","host":"localhost","port":3306,"principal":"","javaSecurityKrb5Conf":"","loginUserKeytabUsername":"","loginUserKeytabPath":"","userName":"root","password":"xxx","database":"ds","connectType":"","other":{"serverTimezone":"GMT-8"},"id":2}
      * @return create result code
      */
     @Operation(summary = "createDataSource", description = "CREATE_DATA_SOURCE_NOTES")
@@ -102,7 +102,7 @@ public Result createDataSource(@Parameter(hidden = true) @RequestAttribute(value
      * @param loginUser login user
      * @param id        datasource id
      * @param jsonStr   datasource param
-     *                  example: {"type":"MYSQL","name":"txx","note":"","host":"localhost","port":3306,"principal":"","javaSecurityKrb5Conf":"","loginUserKeytabUsername":"","loginUserKeytabPath":"","userName":"root","password":"xxx","database":"ds","connectType":"","other":{"serverTimezone":"GMT-8"},"id":2,"testFlag":0,"bindTestId":1}
+     *                  example: {"type":"MYSQL","name":"txx","note":"","host":"localhost","port":3306,"principal":"","javaSecurityKrb5Conf":"","loginUserKeytabUsername":"","loginUserKeytabPath":"","userName":"root","password":"xxx","database":"ds","connectType":"","other":{"serverTimezone":"GMT-8"},"id":2}
      * @return update result code
      */
     @Operation(summary = "updateDataSource", description = "UPDATE_DATA_SOURCE_NOTES")
@@ -146,7 +146,7 @@ public Result queryDataSource(@Parameter(hidden = true) @RequestAttribute(value
     }
 
     /**
-     * query online/testDatasource by type
+     * query datasource by type
      *
      * @param loginUser login user
      * @param type data source type
@@ -155,16 +155,14 @@ public Result queryDataSource(@Parameter(hidden = true) @RequestAttribute(value
     @Operation(summary = "queryDataSourceList", description = "QUERY_DATA_SOURCE_LIST_BY_TYPE_NOTES")
     @Parameters({
             @Parameter(name = "type", description = "DB_TYPE", required = true, schema = @Schema(implementation = DbType.class)),
-            @Parameter(name = "testFlag", description = "DB_TEST_FLAG", required = true, schema = @Schema(implementation = int.class))
     })
     @GetMapping(value = "/list")
     @ResponseStatus(HttpStatus.OK)
     @ApiException(QUERY_DATASOURCE_ERROR)
     @AccessLogAnnotation(ignoreRequestArgs = "loginUser")
     public Result queryDataSourceList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
-                                      @RequestParam("type") DbType type,
-                                      @RequestParam("testFlag") int testFlag) {
-        Map<String, Object> result = dataSourceService.queryDataSourceList(loginUser, type.ordinal(), testFlag);
+                                      @RequestParam("type") DbType type) {
+        Map<String, Object> result = dataSourceService.queryDataSourceList(loginUser, type.ordinal());
         return returnDataList(result);
     }
 

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java

@@ -69,13 +69,13 @@ public interface DataSourceService {
     Result queryDataSourceListPaging(User loginUser, String searchVal, Integer pageNo, Integer pageSize);
 
     /**
-     * query online/test data resource list
+     * query data resource list
      *
      * @param loginUser login user
      * @param type      data source type
      * @return data source list page
      */
-    Map<String, Object> queryDataSourceList(User loginUser, Integer type, int testFlag);
+    Map<String, Object> queryDataSourceList(User loginUser, Integer type);
 
     /**
      * verify datasource exists

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java

@@ -133,8 +133,6 @@ public Result<Object> createDataSource(User loginUser, BaseDataSourceParamDTO da
         dataSource.setConnectionParams(JSONUtils.toJsonString(connectionParam));
         dataSource.setCreateTime(now);
         dataSource.setUpdateTime(now);
-        dataSource.setTestFlag(datasourceParam.getTestFlag());
-        dataSource.setBindTestId(datasourceParam.getBindTestId());
         try {
             dataSourceMapper.insert(dataSource);
             putMsg(result, Status.SUCCESS);
@@ -205,11 +203,6 @@ public Result<Object> updateDataSource(int id, User loginUser, BaseDataSourcePar
         dataSource.setType(dataSource.getType());
         dataSource.setConnectionParams(JSONUtils.toJsonString(connectionParam));
         dataSource.setUpdateTime(now);
-        if (dataSource.getTestFlag() == 1 && dataSourceParam.getTestFlag() == 0) {
-            clearBindTestId(id);
-        }
-        dataSource.setTestFlag(dataSourceParam.getTestFlag());
-        dataSource.setBindTestId(dataSourceParam.getBindTestId());
         try {
             dataSourceMapper.updateById(dataSource);
             log.info("Update datasource complete, datasourceId:{}, datasourceName:{}.", dataSource.getId(),
@@ -257,8 +250,7 @@ public Map<String, Object> queryDataSource(int id, User loginUser) {
         baseDataSourceParamDTO.setId(dataSource.getId());
         baseDataSourceParamDTO.setName(dataSource.getName());
         baseDataSourceParamDTO.setNote(dataSource.getNote());
-        baseDataSourceParamDTO.setTestFlag(dataSource.getTestFlag());
-        baseDataSourceParamDTO.setBindTestId(dataSource.getBindTestId());
+
         result.put(Constants.DATA_LIST, baseDataSourceParamDTO);
         putMsg(result, Status.SUCCESS);
         return result;
@@ -330,12 +322,12 @@ private String getHiddenPassword() {
      * @return data source list page
      */
     @Override
-    public Map<String, Object> queryDataSourceList(User loginUser, Integer type, int testFlag) {
+    public Map<String, Object> queryDataSourceList(User loginUser, Integer type) {
         Map<String, Object> result = new HashMap<>();
 
         List<DataSource> datasourceList = null;
         if (loginUser.getUserType().equals(UserType.ADMIN_USER)) {
-            datasourceList = dataSourceMapper.queryDataSourceByType(0, type, testFlag);
+            datasourceList = dataSourceMapper.queryDataSourceByType(0, type);
         } else {
             Set<Integer> ids = resourcePermissionCheckService
                     .userOwnedResourceIdsAcquisition(AuthorizationType.DATASOURCE, loginUser.getId(), log);
@@ -345,8 +337,7 @@ public Map<String, Object> queryDataSourceList(User loginUser, Integer type, int
                 return result;
             }
             datasourceList = dataSourceMapper.selectBatchIds(ids).stream()
-                    .filter(dataSource -> dataSource.getType().getCode() == type)
-                    .filter(dataSource -> dataSource.getTestFlag() == testFlag).collect(Collectors.toList());
+                    .filter(dataSource -> dataSource.getType().getCode() == type).collect(Collectors.toList());
         }
         result.put(Constants.DATA_LIST, datasourceList);
         putMsg(result, Status.SUCCESS);
@@ -460,7 +451,6 @@ public Result<Object> delete(User loginUser, int datasourceId) {
             }
             dataSourceMapper.deleteById(datasourceId);
             datasourceUserMapper.deleteByDatasourceId(datasourceId);
-            clearBindTestId(datasourceId);
             log.info("Delete datasource complete, datasourceId:{}.", datasourceId);
             putMsg(result, Status.SUCCESS);
         } catch (Exception e) {
@@ -709,8 +699,5 @@ private static void closeResult(ResultSet rs) {
             }
         }
     }
-    private void clearBindTestId(Integer bindTestId) {
-        dataSourceMapper.clearBindTestId(bindTestId);
-    }
 
 }

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java

@@ -319,7 +319,7 @@ public void queryDataSourceListTest() {
         dataSource.setType(DbType.MYSQL);
         Mockito.when(dataSourceMapper.selectBatchIds(dataSourceIds)).thenReturn(Collections.singletonList(dataSource));
         Map<String, Object> map =
-                dataSourceService.queryDataSourceList(loginUser, DbType.MYSQL.ordinal(), Constants.TEST_FLAG_NO);
+                dataSourceService.queryDataSourceList(loginUser, DbType.MYSQL.ordinal());
         Assertions.assertEquals(Status.SUCCESS, map.get(Constants.STATUS));
     }
 

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java

@@ -24,7 +24,6 @@
 
 import lombok.Data;
 
-import com.baomidou.mybatisplus.annotation.FieldFill;
 import com.baomidou.mybatisplus.annotation.IdType;
 import com.baomidou.mybatisplus.annotation.TableField;
 import com.baomidou.mybatisplus.annotation.TableId;
@@ -81,17 +80,6 @@ public class DataSource {
      */
     private Date updateTime;
 
-    /**
-     * test flag
-     */
-    protected int testFlag;
-
-    /**
-     * bind test data source id
-     */
-    @TableField(fill = FieldFill.INSERT_UPDATE)
-    protected Integer bindTestId;
-
     @Override
     public boolean equals(Object o) {
         if (this == o) {

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java

@@ -33,13 +33,12 @@
 public interface DataSourceMapper extends BaseMapper<DataSource> {
 
     /**
-     * query online/testDatasource by type
+     * query datasource by type
      * @param userId userId
      * @param type type
      * @return datasource list
      */
-    List<DataSource> queryDataSourceByType(@Param("userId") int userId, @Param("type") Integer type,
-                                           @Param("testFlag") int testFlag);
+    List<DataSource> queryDataSourceByType(@Param("userId") int userId, @Param("type") Integer type);
 
     /**
      * datasource page
@@ -110,18 +109,4 @@ <T> List<DataSource> listAuthorizedDataSource(@Param("userId") int userId,
     IPage<DataSource> selectPagingByIds(Page<DataSource> dataSourcePage,
                                         @Param("dataSourceIds") List<Integer> dataSourceIds,
                                         @Param("name") String name);
-
-    /**
-     * clearBindTestId
-     * @param bindTestId
-     * @return
-     */
-    void clearBindTestId(@Param("bindTestId") Integer bindTestId);
-
-    /**
-     * queryTestDataSourceId
-     * @param onlineDataSourceId
-     * @return Integer
-     */
-    Integer queryTestDataSourceId(@Param("dataSourceId") Integer onlineDataSourceId);
 }

dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.xml

@@ -26,7 +26,6 @@
         <include refid="baseSql"/>
         from t_ds_datasource
         where type=#{type}
-        and test_flag=#{testFlag}
         <if test="userId != 0">
             and id in
             (select datasource_id
@@ -41,7 +40,7 @@
 
     <select id="selectPaging" resultType="org.apache.dolphinscheduler.dao.entity.DataSource">
         select
-        d.id, d.name, d.note, d.type, d.user_id, connection_params, d.create_time, d.update_time, d.test_flag, d.bind_test_id
+        d.id, d.name, d.note, d.type, d.user_id, connection_params, d.create_time, d.update_time
         ,
         u.user_name as user_name
         from t_ds_datasource d
@@ -69,7 +68,7 @@
         where name=#{name}
     </select>
     <select id="queryAuthedDatasource" resultType="org.apache.dolphinscheduler.dao.entity.DataSource">
-        select ds.id, ds.name, ds.note, ds.type, ds.user_id, ds.connection_params, ds.create_time, ds.update_time, ds.test_flag, ds.bind_test_id
+        select ds.id, ds.name, ds.note, ds.type, ds.user_id, ds.connection_params, ds.create_time, ds.update_time
         from t_ds_datasource ds, t_ds_relation_datasource_user rel
         where ds.id = rel.datasource_id AND rel.user_id = #{userId}
     </select>
@@ -116,7 +115,7 @@
 
     <select id="selectPagingByIds" resultType="org.apache.dolphinscheduler.dao.entity.DataSource">
         select
-        d.id, d.name, d.note, d.type, d.user_id, connection_params, d.create_time, d.update_time, d.test_flag, d.bind_test_id
+        d.id, d.name, d.note, d.type, d.user_id, connection_params, d.create_time, d.update_time
         ,
         u.user_name as user_name
         from t_ds_datasource d
@@ -133,12 +132,5 @@
         </if>
         order by update_time desc
     </select>
-    <select id="queryTestDataSourceId" resultType="java.lang.Integer">
-        select d.bind_test_id
-        from t_ds_datasource d
-        where d.id = #{dataSourceId}
-    </select>
-    <update id="clearBindTestId">
-        update t_ds_datasource d set bind_test_id=null where bind_test_id=#{bindTestId}
-    </update>
+
 </mapper>

dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql

@@ -361,8 +361,6 @@ CREATE TABLE t_ds_datasource
     connection_params text        NOT NULL,
     create_time       datetime    NOT NULL,
     update_time       datetime     DEFAULT NULL,
-    test_flag           int DEFAULT NULL,
-    bind_test_id        int DEFAULT NULL,
     PRIMARY KEY (id),
     UNIQUE KEY t_ds_datasource_name_un (name, type)
 );

dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql

@@ -366,8 +366,6 @@ CREATE TABLE `t_ds_datasource` (
   `connection_params` text NOT NULL COMMENT 'json connection params',
   `create_time` datetime NOT NULL COMMENT 'create time',
   `update_time` datetime DEFAULT NULL COMMENT 'update time',
-  `test_flag`   tinyint(4) DEFAULT NULL COMMENT 'test flag：0 normal, 1 testDataSource',
-  `bind_test_id` int(11)  DEFAULT NULL COMMENT 'bind testDataSource id',
   PRIMARY KEY (`id`),
   UNIQUE KEY `t_ds_datasource_name_un` (`name`, `type`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin;

dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql

@@ -289,8 +289,6 @@ CREATE TABLE t_ds_datasource (
   connection_params text NOT NULL ,
   create_time timestamp NOT NULL ,
   update_time timestamp DEFAULT NULL ,
-  test_flag   int DEFAULT NULL ,
-  bind_test_id   int DEFAULT NULL ,
   PRIMARY KEY (id),
   CONSTRAINT t_ds_datasource_name_un UNIQUE (name, type)
 ) ;

dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql

@@ -56,30 +56,6 @@ delimiter ;
 CALL uc_dolphin_T_t_ds_error_command_R_test_flag;
 DROP PROCEDURE uc_dolphin_T_t_ds_error_command_R_test_flag;
 
--- uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id
-drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id;
-delimiter d//
-CREATE PROCEDURE uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id()
-BEGIN
-       IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-           WHERE TABLE_NAME='t_ds_datasource'
-           AND TABLE_SCHEMA=(SELECT DATABASE())
-           AND COLUMN_NAME ='test_flag')
-           and NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-           WHERE TABLE_NAME='t_ds_datasource'
-           AND TABLE_SCHEMA=(SELECT DATABASE())
-           AND COLUMN_NAME ='bind_test_id')
-   THEN
-ALTER TABLE t_ds_datasource ADD `test_flag` tinyint(4) DEFAULT null COMMENT 'test flag：0 normal, 1 testDataSource';
-ALTER TABLE t_ds_datasource ADD `bind_test_id` int DEFAULT null COMMENT 'bind testDataSource id';
-END IF;
-END;
-
-d//
-
-delimiter ;
-CALL uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id;
-DROP PROCEDURE uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id;
 
 -- uc_dolphin_T_t_ds_process_instance_R_test_flag
 drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_process_instance_R_test_flag;

dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl.sql

@@ -58,30 +58,6 @@ delimiter ;
 select uc_dolphin_T_t_ds_error_command_R_test_flag();
 DROP FUNCTION uc_dolphin_T_t_ds_error_command_R_test_flag();
 
--- uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id
-delimiter ;
-DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id();
-delimiter d//
-CREATE FUNCTION uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id() RETURNS void AS $$
-BEGIN
-       IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-          WHERE TABLE_CATALOG=current_database()
-          AND TABLE_SCHEMA=current_schema()
-          AND TABLE_NAME='t_ds_datasource'
-          AND COLUMN_NAME ='test_flag')
-      THEN
-ALTER TABLE t_ds_datasource alter column test_flag type int;
-ALTER TABLE t_ds_datasource alter column test_flag set DEFAULT NULL;
-ALTER TABLE t_ds_datasource alter column bind_test_id type int;
-ALTER TABLE t_ds_datasource alter column bind_test_id set DEFAULT NULL;
-END IF;
-END;
-$$ LANGUAGE plpgsql;
-d//
-delimiter ;
-select uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id();
-DROP FUNCTION uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id();
-
 -- uc_dolphin_T_t_ds_process_instance_R_test_flag
 delimiter ;
 DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_process_instance_R_test_flag();

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java

@@ -19,7 +19,6 @@
 
 import static java.util.stream.Collectors.toList;
 
-import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.enums.UserType;
 import org.apache.dolphinscheduler.common.utils.DateUtils;
 import org.apache.dolphinscheduler.dao.BaseDaoTest;
@@ -126,7 +125,7 @@ public void testQueryDataSourceByType() {
         Map<Integer, DataSource> datasourceMap = createDataSourceMap(userId, "test");
 
         List<DataSource> actualDataSources = dataSourceMapper.queryDataSourceByType(
-                0, DbType.MYSQL.ordinal(), Constants.TEST_FLAG_NO);
+                0, DbType.MYSQL.ordinal());
 
         Assertions.assertTrue(actualDataSources.size() >= 2);
 
@@ -380,7 +379,6 @@ private DataSource createDataSource(Integer userId, String name) {
         dataSource.setType(DbType.MYSQL);
         dataSource.setNote("mysql test");
         dataSource.setConnectionParams("hello mysql");
-        dataSource.setTestFlag(Constants.TEST_FLAG_NO);
         dataSource.setUpdateTime(DateUtils.getCurrentDate());
         dataSource.setCreateTime(DateUtils.getCurrentDate());