apache · zhongjiajie · Mar 15, 2023 · Feb 20, 2023
@@ -99,7 +99,7 @@ conf:
     # user data local directory path, please make sure the directory exists and have read write permissions
     data.basedir.path: /tmp/dolphinscheduler
 
-    # resource storage type: HDFS, S3, NONE
+    # resource storage type: HDFS, S3, OSS, GCS, NONE
     resource.storage.type: S3
 
     # resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended

@@ -197,7 +197,7 @@ The default configuration is as follows:
 | Parameters | Default value | Description |
 |--|--|--|
 |data.basedir.path | /tmp/dolphinscheduler | local directory used to store temp files|
-|resource.storage.type | NONE | type of resource files: HDFS, S3, NONE|
+|resource.storage.type | NONE | type of resource files: HDFS, S3, OSS, GCS, NONE|
 |resource.upload.path | /dolphinscheduler | storage path of resource files|
 |aws.access.key.id | minioadmin | access key id of S3|
 |aws.secret.access.key | minioadmin | secret access key of S3|

@@ -553,7 +553,7 @@ common:
 |                                                                      |                                                                                                                               |                                       |
 | `common.configmap.DOLPHINSCHEDULER_OPTS`                             | The jvm options for dolphinscheduler, suitable for all servers                                                                | `""`                                  |
 | `common.configmap.DATA_BASEDIR_PATH`                                 | User data directory path, self configuration, please make sure the directory exists and have read write permissions           | `/tmp/dolphinscheduler`               |
-| `common.configmap.RESOURCE_STORAGE_TYPE`                             | Resource storage type: HDFS, S3, NONE                                                                                         | `HDFS`                                |
+| `common.configmap.RESOURCE_STORAGE_TYPE`                             | Resource storage type: HDFS, S3, OSS, GCS, NONE                                                                               | `HDFS`                                |
 | `common.configmap.RESOURCE_UPLOAD_PATH`                              | Resource store on HDFS/S3 path, please make sure the directory exists on hdfs and have read write permissions                 | `/dolphinscheduler`                   |
 | `common.configmap.FS_DEFAULT_FS`                                     | Resource storage file system like `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler`                             | `file:///`                            |
 | `common.configmap.FS_S3A_ENDPOINT`                                   | S3 endpoint when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3`                                                      | `s3.xxx.amazonaws.com`                |

@@ -54,7 +54,7 @@ data.basedir.path=/tmp/dolphinscheduler
 # resource view suffixs
 #resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
 
-# resource storage type: HDFS, S3, OSS, NONE
+# resource storage type: HDFS, S3, OSS, GCS, NONE
 resource.storage.type=NONE
 # resource store on HDFS/S3/OSS path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
 resource.storage.upload.base.path=/tmp/dolphinscheduler

@@ -194,7 +194,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
 | 参数 | 默认值 | 描述 |
 |--|--|--|
 |data.basedir.path | /tmp/dolphinscheduler | 本地工作目录,用于存放临时文件|
-|resource.storage.type | NONE | 资源文件存储类型: HDFS,S3,NONE|
+|resource.storage.type | NONE | 资源文件存储类型: HDFS,S3,OSS,GCS,NONE|
 |resource.upload.path | /dolphinscheduler | 资源文件存储路径|
 |aws.access.key.id | minioadmin | S3 access key|
 |aws.secret.access.key | minioadmin | S3 secret access key|

@@ -551,7 +551,7 @@ common:
 |                                                                      |                                                                                                                               |                                       |
 | `common.configmap.DOLPHINSCHEDULER_OPTS`                             | The jvm options for dolphinscheduler, suitable for all servers                                                                | `""`                                  |
 | `common.configmap.DATA_BASEDIR_PATH`                                 | User data directory path, self configuration, please make sure the directory exists and have read write permissions           | `/tmp/dolphinscheduler`               |
-| `common.configmap.RESOURCE_STORAGE_TYPE`                             | Resource storage type: HDFS, S3, NONE                                                                                         | `HDFS`                                |
+| `common.configmap.RESOURCE_STORAGE_TYPE`                             | Resource storage type: HDFS, S3, OSS, GCS, NONE                                                                               | `HDFS`                                |
 | `common.configmap.RESOURCE_UPLOAD_PATH`                              | Resource store on HDFS/S3 path, please make sure the directory exists on hdfs and have read write permissions                 | `/dolphinscheduler`                   |
 | `common.configmap.FS_DEFAULT_FS`                                     | Resource storage file system like `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler`                             | `file:///`                            |
 | `common.configmap.FS_S3A_ENDPOINT`                                   | S3 endpoint when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3`                                                      | `s3.xxx.amazonaws.com`                |

@@ -53,7 +53,7 @@ Dolphinscheduler 资源中心使用本地系统默认是开启的，不需要用
 # user data local directory path, please make sure the directory exists and have read write permissions
 data.basedir.path=/tmp/dolphinscheduler
 
-# resource storage type: LOCAL, HDFS, S3, OSS
+# resource storage type: LOCAL, HDFS, S3, OSS, GCS
 resource.storage.type=LOCAL
 
 # resource store on HDFS/S3/OSS path, resource file will store to this hadoop hdfs path, self configuration,

@@ -21,7 +21,7 @@ data.basedir.path=/tmp/dolphinscheduler
 # resource view suffixs
 #resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
 
-# resource storage type: HDFS, S3, OSS, NONE
+# resource storage type: HDFS, S3, OSS, GCS, NONE
 resource.storage.type=S3
 # resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
 resource.storage.upload.base.path=/dolphinscheduler

@@ -105,6 +105,7 @@
         <aws-java-sdk-redshift.version>1.12.300</aws-java-sdk-redshift.version>
         <azure-resourcemanager.version>2.21.0</azure-resourcemanager.version>
         <azure-resourcemanager-datafactory.version>1.0.0-beta.19</azure-resourcemanager-datafactory.version>
+        <google-cloud-storage.version>2.18.0</google-cloud-storage.version>
     </properties>
 
     <dependencyManagement>
@@ -670,6 +671,12 @@
                 <version>${aliyun-sdk-oss.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>com.google.cloud</groupId>
+                <artifactId>google-cloud-storage</artifactId>
+                <version>${google-cloud-storage.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>joda-time</groupId>
                 <artifactId>joda-time</artifactId>

@@ -143,6 +143,10 @@ private Constants() {
     public static final String ALIBABA_CLOUD_OSS_BUCKET_NAME = "resource.alibaba.cloud.oss.bucket.name";
     public static final String ALIBABA_CLOUD_OSS_END_POINT = "resource.alibaba.cloud.oss.endpoint";
 
+    public static final String GOOGLE_CLOUD_STORAGE_BUCKET_NAME = "resource.google.cloud.storage.bucket.name";
+
+    public static final String GOOGLE_CLOUD_STORAGE_CREDENTIAL = "resource.google.cloud.storage.credential";
+
     /**
      * fetch applicationId way
      */

@@ -21,5 +21,5 @@
  * data base types
  */
 public enum ResUploadType {
-    LOCAL, HDFS, S3, OSS, NONE
+    LOCAL, HDFS, S3, OSS, GCS, NONE
 }
@@ -21,7 +21,7 @@ data.basedir.path=/tmp/dolphinscheduler
 # resource view suffixs
 #resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
 
-# resource storage type: LOCAL, HDFS, S3, OSS, NONE. LOCAL type is a specific type of HDFS with "resource.hdfs.fs.defaultFS = file:///" configuration
+# resource storage type: LOCAL, HDFS, S3, OSS, GCS, NONE. LOCAL type is a specific type of HDFS with "resource.hdfs.fs.defaultFS = file:///" configuration
 # please notice that LOCAL mode does not support reading and writing in distributed mode, which mean you can only use your resource in one machine, unless
 # use shared file mount point
 resource.storage.type=LOCAL
@@ -61,6 +61,11 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
 # oss bucket endpoint, required if you set resource.storage.type=OSS
 resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
 
+# the location of the google cloud credential, required if you set resource.storage.type=GCS
+resource.google.cloud.storage.credential=/path/to/credential
+# gcs bucket name, required if you set resource.storage.type=GCS
+resource.google.cloud.storage.bucket.name=<your-bucket>
+
 # if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
 resource.hdfs.root.user=hdfs
 # if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir

@@ -519,6 +519,35 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
     DmJdbcDriver18 8.1.2.79: https://mvnrepository.com/artifact/com.dameng/DmJdbcDriver18/8.1.2.79, Apache 2.0
     nimbus-jose-jwt 9.22: https://mvnrepository.com/artifact/com.nimbusds/nimbus-jose-jwt/9.22, Apache 2.0
     woodstox-core 6.4.0: https://mvnrepository.com/artifact/com.fasterxml.woodstox/woodstox-core/6.4.0, Apache 2.0
+    auto-value 1.10.1: https://mvnrepository.com/artifact/com.google.auto.value/auto-value/1.10.1, Apache 2.0
+    auto-value-annotations 1.10.1: https://mvnrepository.com/artifact/com.google.auto.value/auto-value-annotations/1.10.1, Apache 2.0
+    conscrypt-openjdk-uber 2.5.2: https://mvnrepository.com/artifact/org.conscrypt/conscrypt-openjdk-uber/2.5.2, Apache 2.0
+    gapic-google-cloud-storage-v2 2.18.0-alpha: https://mvnrepository.com/artifact/com.google.api.grpc/gapic-google-cloud-storage-v2/2.18.0-alpha, Apache 2.0
+    google-api-client 2.2.0: https://mvnrepository.com/artifact/com.google.api-client/google-api-client/2.2.0, Apache 2.0
+    google-api-services-storage v1-rev20220705-2.0.0: https://mvnrepository.com/artifact/com.google.apis/google-api-services-storage/v1-rev20220705-2.0.0, Apache 2.0
+    google-cloud-core 2.10.0: https://mvnrepository.com/artifact/com.google.cloud/google-cloud-core/2.10.0, Apache 2.0
+    google-cloud-core-grpc 2.10.0: https://mvnrepository.com/artifact/com.google.cloud/google-cloud-core-grpc/2.10.0, Apache 2.0
+    google-cloud-core-http 2.10.0: https://mvnrepository.com/artifact/com.google.cloud/google-cloud-core-http/2.10.0, Apache 2.0
+    google-cloud-storage 2.18.0: https://mvnrepository.com/artifact/com.google.cloud/google-cloud-storage/2.18.0, Apache 2.0
+    google-http-client 1.42.3: https://mvnrepository.com/artifact/com.google.http-client/google-http-client/1.42.3 Apache 2.0
+    google-http-client-apache-v2 1.42.3: https://mvnrepository.com/artifact/com.google.http-client/google-http-client-apache-v2/1.42.3, Apache 2.0
+    google-http-client-appengine 1.42.3: https://mvnrepository.com/artifact/com.google.http-client/google-http-client-appengine/1.42.3, Apache 2.0
+    google-http-client-gson 1.42.3: https://mvnrepository.com/artifact/com.google.http-client/google-http-client-gson/1.42.3, Apache 2.0
+    google-http-client-jackson2 1.42.3: https://mvnrepository.com/artifact/com.google.http-client/google-http-client-jackson2/1.42.3, Apache 2.0
+    google-oauth-client 1.34.1: https://mvnrepository.com/artifact/com.google.oauth-client/google-oauth-client/1.34.1, Apache 2.0
+    grpc-alts 1.41.0: https://mvnrepository.com/artifact/io.grpc/grpc-alts/1.41.0, Apache 2.0
+    grpc-auth 1.41.0: https://mvnrepository.com/artifact/io.grpc/grpc-auth/1.41.0, Apache 2.0
+    grpc-google-cloud-storage-v2 2.18.0-alpha: https://mvnrepository.com/artifact/com.google.api.grpc/grpc-google-cloud-storage-v2/2.18.0-alpha, Apache 2.0
+    grpc-googleapis 1.52.1: https://mvnrepository.com/artifact/io.grpc/grpc-googleapis/1.52.1, Apache 2.0
+    grpc-netty-shaded 1.41.0: https://mvnrepository.com/artifact/io.grpc/grpc-netty-shaded/1.41.0, Apache 2.0
+    grpc-services 1.41.0: https://mvnrepository.com/artifact/io.grpc/grpc-services/1.41.0, Apache 2.0
+    grpc-xds 1.41.0: https://mvnrepository.com/artifact/io.grpc/grpc-xds/1.41.0, Apache 2.0
+    opencensus-api 0.31.1: https://mvnrepository.com/artifact/io.opencensus/opencensus-api/0.31.1, Apache 2.0
+    opencensus-contrib-http-util 0.31.1: https://mvnrepository.com/artifact/io.opencensus/opencensus-contrib-http-util/0.31.1, Apache 2.0
+    opencensus-proto 0.2.0: https://mvnrepository.com/artifact/io.opencensus/opencensus-proto/0.2.0, Apache 2.0
+    proto-google-cloud-storage-v2 2.18.0-alpha: https://mvnrepository.com/artifact/com.google.api.grpc/proto-google-cloud-storage-v2/2.18.0-alpha, Apache 2.0
+    proto-google-iam-v1 1.9.0: https://mvnrepository.com/artifact/com.google.api.grpc/proto-google-iam-v1/1.9.0, Apache 2.0
+
 
 jna-platform
 
@@ -552,6 +581,13 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
     automaton 1.11-8 https://mvnrepository.com/artifact/dk.brics.automaton/automaton/1.11-8, BSD 2-clause
     protobuf-java 3.17.2: https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java/3.17.2 BSD 3-clause
     protobuf-java-util 3.17.2: https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java-util/3.17.2 BSD 3-clause
+    api-common 2.6.0: https://mvnrepository.com/artifact/com.google.api/api-common/2.6.0, BSD
+    gax 2.23.0: https://mvnrepository.com/artifact/com.google.api/gax/2.23.0, BSD
+    gax-grpc 2.23.0: https://mvnrepository.com/artifact/com.google.api/gax-grpc/2.23.0, BSD
+    gax-httpjson 0.108.0: https://mvnrepository.com/artifact/com.google.api/gax-httpjson/0.108.0, BSD
+    google-auth-library-credentials 1.15.0: https://mvnrepository.com/artifact/com.google.auth/google-auth-library-credentials/1.15.0, BSD 3-clause
+    google-auth-library-oauth2-http 1.15.0: https://mvnrepository.com/artifact/com.google.auth/google-auth-library-oauth2-http/1.15.0, BSD 3-clause
+    threetenbp 1.6.5: https://mvnrepository.com/artifact/org.threeten/threetenbp/1.6.5, BSD 3-clause
 
 ========================================================================
 CDDL licenses

@@ -0,0 +1,26 @@
+Copyright 2016, Google Inc.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.