Skip to content

Commit

Permalink
JDBC connector spark_options should provide all the properties (#214)
Browse files Browse the repository at this point in the history
We allow users to configure additional properties for their JDBC
storage connector to control performance, driver and so on.

These properties needs to be included in the spark_options method
  • Loading branch information
SirOibaf committed Jan 11, 2021
1 parent 832c115 commit 880366e
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ public Map<String, String> getSparkOptions() throws FeatureStoreException {
}

@JsonIgnore
private Map<String, String> getJdbcOptions() throws FeatureStoreException {
private Map<String, String> getJdbcOptions() {
Map<String, String> options = Arrays.stream(arguments.split(","))
.map(arg -> arg.split("="))
.collect(Collectors.toMap(a -> a[0], a -> a[1]));
Expand Down
12 changes: 5 additions & 7 deletions python/hsfs/storage_connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -322,8 +322,7 @@ def arguments(self):

@property
def path(self):
"""If the connector refers to a path (e.g. S3) - return the path of the connector
"""
"""If the connector refers to a path (e.g. S3) - return the path of the connector"""
if self._storage_connector_type.upper() == "S3":
return "s3://" + self._bucket
else:
Expand All @@ -338,11 +337,10 @@ def spark_options(self):
if self._storage_connector_type.upper() == "JDBC":
args = [arg.split("=") for arg in self._arguments.split(",")]

return {
"url": self._connection_string,
"user": [arg[1] for arg in args if arg[0] == "user"][0],
"password": [arg[1] for arg in args if arg[0] == "password"][0],
}
options = {a[0]: a[1] for a in args}
options["url"] = self._connection_string

return options
elif self._storage_connector_type.upper() == "REDSHIFT":
connstr = (
"jdbc:redshift://"
Expand Down

0 comments on commit 880366e

Please sign in to comment.