Add support for exchange spooling on Azure Blob Storage

trinodb · May 10, 2022 · 6ed6160 · 6ed6160
1 parent fef263e
commit 6ed6160
Show file tree

Hide file tree

Showing 5 changed files with 747 additions and 0 deletions.
diff --git a/plugin/trino-exchange-filesystem/pom.xml b/plugin/trino-exchange-filesystem/pom.xml
@@ -15,10 +15,18 @@
     <properties>
         <air.main.basedir>${project.parent.basedir}</air.main.basedir>
         <awsjavasdk.version>2.17.151</awsjavasdk.version>
+        <azurejavasdk.version>1.1.1</azurejavasdk.version>
     </properties>
 
     <dependencyManagement>
         <dependencies>
+            <dependency>
+                <groupId>com.azure</groupId>
+                <artifactId>azure-sdk-bom</artifactId>
+                <type>pom</type>
+                <version>${azurejavasdk.version}</version>
+                <scope>import</scope>
+            </dependency>
             <dependency>
                 <groupId>software.amazon.awssdk</groupId>
                 <artifactId>bom</artifactId>
@@ -65,6 +73,44 @@
             <artifactId>units</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.azure</groupId>
+            <artifactId>azure-core</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.azure</groupId>
+            <artifactId>azure-identity</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.github.stephenc.jcip</groupId>
+                    <artifactId>jcip-annotations</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.projectlombok</groupId>
+                    <artifactId>lombok</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>net.java.dev.jna</groupId>
+                    <artifactId>jna-platform</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>com.azure</groupId>
+            <artifactId>azure-storage-blob</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.azure</groupId>
+            <artifactId>azure-storage-blob-batch</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.google.code.findbugs</groupId>
             <artifactId>jsr305</artifactId>
@@ -80,6 +126,12 @@
             <artifactId>guice</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>io.projectreactor</groupId>
+            <artifactId>reactor-core</artifactId>
+            <version>3.4.13</version>
+        </dependency>
+
         <dependency>
             <groupId>javax.annotation</groupId>
             <artifactId>javax.annotation-api</artifactId>

diff --git a/...ilesystem/src/main/java/io/trino/plugin/exchange/filesystem/FileSystemExchangeModule.java b/...ilesystem/src/main/java/io/trino/plugin/exchange/filesystem/FileSystemExchangeModule.java
@@ -17,6 +17,8 @@
 import com.google.inject.Binder;
 import com.google.inject.Scopes;
 import io.airlift.configuration.AbstractConfigurationAwareModule;
+import io.trino.plugin.exchange.filesystem.azure.AzureBlobFileSystemExchangeStorage;
+import io.trino.plugin.exchange.filesystem.azure.ExchangeAzureConfig;
 import io.trino.plugin.exchange.filesystem.local.LocalFileSystemExchangeStorage;
 import io.trino.plugin.exchange.filesystem.s3.ExchangeS3Config;
 import io.trino.plugin.exchange.filesystem.s3.S3FileSystemExchangeStorage;
@@ -57,6 +59,10 @@ else if (ImmutableSet.of("s3", "s3a", "s3n").contains(scheme)) {
             binder.bind(FileSystemExchangeStorage.class).to(S3FileSystemExchangeStorage.class).in(Scopes.SINGLETON);
             configBinder(binder).bindConfig(ExchangeS3Config.class);
         }
+        else if (ImmutableSet.of("abfs", "abfss").contains(scheme)) {
+            binder.bind(FileSystemExchangeStorage.class).to(AzureBlobFileSystemExchangeStorage.class).in(Scopes.SINGLETON);
+            configBinder(binder).bindConfig(ExchangeAzureConfig.class);
+        }
         else {
             throw new TrinoException(NOT_SUPPORTED, format("Scheme %s is not supported as exchange spooling storage", scheme));
         }