[Improve][Connector-V2][OSS-Jindo] Optimize jindo oss connector (#4964)

* [Improve][Connector-V2][Jindo-Oss] Optimize jindo-oss connector * [Improve][Connector-V2][Jindo-Oss] Update module name * [Hotfix][Connector-V2][StarRocks] Fix code style
apache · Jun 28, 2023 · 5fbfd05 · 5fbfd05
1 parent 526197a
commit 5fbfd05
Show file tree

Hide file tree

Showing 12 changed files with 6 additions and 4 deletions.
diff --git a/...tor-file/connector-file-oss-jindo/pom.xml → ...tor-file/connector-file-jindo-oss/pom.xml b/...tor-file/connector-file-oss-jindo/pom.xml → ...tor-file/connector-file-jindo-oss/pom.xml
@@ -26,8 +26,8 @@
         <version>${revision}</version>
     </parent>
 
-    <artifactId>connector-file-oss-jindo</artifactId>
-    <name>SeaTunnel : Connectors V2 : File : Oss Jindo</name>
+    <artifactId>connector-file-jindo-oss</artifactId>
+    <name>SeaTunnel : Connectors V2 : File : Jindo Oss</name>
 
     <properties>
         <jindo-sdk.version>4.6.1</jindo-sdk.version>

diff --git a/...rs/seatunnel/file/oss/config/OssConf.java → ...rs/seatunnel/file/oss/config/OssConf.java b/...rs/seatunnel/file/oss/config/OssConf.java → ...rs/seatunnel/file/oss/config/OssConf.java
@@ -49,6 +49,8 @@ public static HadoopConf buildWithConfig(Config config) {
         ossOptions.put("fs.oss.accessKeyId", config.getString(OssConfig.ACCESS_KEY.key()));
         ossOptions.put("fs.oss.accessKeySecret", config.getString(OssConfig.ACCESS_SECRET.key()));
         ossOptions.put("fs.oss.endpoint", config.getString(OssConfig.ENDPOINT.key()));
+        ossOptions.put("fs.oss.upload.thread.concurrency", "20");
+        ossOptions.put("fs.oss.upload.queue.size", "100");
         hadoopConf.setExtraOptions(ossOptions);
         return hadoopConf;
     }

diff --git a/.../seatunnel/file/oss/config/OssConfig.java → .../seatunnel/file/oss/config/OssConfig.java b/.../seatunnel/file/oss/config/OssConfig.java → .../seatunnel/file/oss/config/OssConfig.java
diff --git a/...exception/OssJindoConnectorException.java → ...exception/OssJindoConnectorException.java b/...exception/OssJindoConnectorException.java → ...exception/OssJindoConnectorException.java
diff --git a/.../seatunnel/file/oss/sink/OssFileSink.java → .../seatunnel/file/oss/sink/OssFileSink.java b/.../seatunnel/file/oss/sink/OssFileSink.java → .../seatunnel/file/oss/sink/OssFileSink.java
diff --git a/...nel/file/oss/sink/OssFileSinkFactory.java → ...nel/file/oss/sink/OssFileSinkFactory.java b/...nel/file/oss/sink/OssFileSinkFactory.java → ...nel/file/oss/sink/OssFileSinkFactory.java
diff --git a/...tunnel/file/oss/source/OssFileSource.java → ...tunnel/file/oss/source/OssFileSource.java b/...tunnel/file/oss/source/OssFileSource.java → ...tunnel/file/oss/source/OssFileSource.java
diff --git a/...file/oss/source/OssFileSourceFactory.java → ...file/oss/source/OssFileSourceFactory.java b/...file/oss/source/OssFileSourceFactory.java → ...file/oss/source/OssFileSourceFactory.java
diff --git a/.../services/org.apache.hadoop.fs.FileSystem → .../services/org.apache.hadoop.fs.FileSystem b/.../services/org.apache.hadoop.fs.FileSystem → .../services/org.apache.hadoop.fs.FileSystem
diff --git a/.../connectors/test/OssJindoFactoryTest.java → .../connectors/test/OssJindoFactoryTest.java b/.../connectors/test/OssJindoFactoryTest.java → .../connectors/test/OssJindoFactoryTest.java
diff --git a/seatunnel-connectors-v2/connector-file/pom.xml b/seatunnel-connectors-v2/connector-file/pom.xml
@@ -38,7 +38,7 @@
         <module>connector-file-base-hadoop</module>
         <module>connector-file-sftp</module>
         <module>connector-file-s3</module>
-        <module>connector-file-oss-jindo</module>
+        <module>connector-file-jindo-oss</module>
     </modules>
 
     <properties>

diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml
@@ -274,7 +274,7 @@
                 </dependency>
                 <dependency>
                     <groupId>org.apache.seatunnel</groupId>
-                    <artifactId>connector-file-oss-jindo</artifactId>
+                    <artifactId>connector-file-jindo-oss</artifactId>
                     <version>${project.version}</version>
                     <scope>provided</scope>
                 </dependency>