Skip to content
This repository has been archived by the owner on Sep 18, 2023. It is now read-only.

Commit

Permalink
[NSE-727] Create table failed with TPCH partiton dataset (#731)
Browse files Browse the repository at this point in the history
  • Loading branch information
zhztheplayer authored Feb 9, 2022
1 parent 07020ba commit 8f81f8e
Showing 1 changed file with 6 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package com.intel.oap.spark.sql.execution.datasources.v2.arrow

import java.net.URI
import java.nio.charset.StandardCharsets
import java.time.ZoneId

import scala.collection.JavaConverters._
Expand Down Expand Up @@ -69,15 +70,15 @@ object ArrowUtils {
}
}

def makeArrowDiscovery(file: String, startOffset: Long, length: Long,
def makeArrowDiscovery(encodedUri: String, startOffset: Long, length: Long,
options: ArrowOptions): FileSystemDatasetFactory = {

val format = getFormat(options)
val allocator = SparkMemoryUtils.contextAllocator()
val factory = new FileSystemDatasetFactory(allocator,
SparkMemoryUtils.contextMemoryPool(),
format,
rewriteUri(file),
rewriteUri(encodedUri),
startOffset,
length)
factory
Expand Down Expand Up @@ -122,8 +123,9 @@ object ArrowUtils {
}
}

private def rewriteUri(uriStr: String): String = {
val uri = URI.create(uriStr)
private def rewriteUri(encodeUri: String): String = {
val decodedUri = java.net.URLDecoder.decode(encodeUri, StandardCharsets.UTF_8.name())
val uri = URI.create(decodedUri)
if (uri.getScheme == "s3" || uri.getScheme == "s3a") {
val s3Rewritten = new URI("s3", uri.getAuthority,
uri.getPath, uri.getQuery, uri.getFragment).toString
Expand Down

0 comments on commit 8f81f8e

Please sign in to comment.