From a9703957b3c90873e45e61da2599e45870031f97 Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Mon, 22 Apr 2019 22:03:25 -0400 Subject: [PATCH 1/7] Add support for Apache Drill --- superset/db_engine_specs.py | 64 +++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/superset/db_engine_specs.py b/superset/db_engine_specs.py index 67aba1264b9e5..da99e878c1b41 100644 --- a/superset/db_engine_specs.py +++ b/superset/db_engine_specs.py @@ -724,6 +724,70 @@ def get_table_names(cls, inspector, schema): return sorted(inspector.get_table_names()) +class DrillEngineSpec(BaseEngineSpec): + """Engine spec for Apache Drill""" + engine = 'drill' + + time_grain_functions = { + None: '{col}', + 'PT1S': "nearestDate({col}, 'SECOND')", + 'PT1M': "nearestDate({col}, 'MINUTE')", + 'PT15M': "nearestDate({col}, 'QUARTER_HOUR')", + 'PT0.5H': "nearestDate({col}, 'HALF_HOUR')", + 'PT1H': "nearestDate({col}, 'HOUR')", + 'P1D': 'TO_DATE({col})', + 'P1W': "nearestDate({col}, 'WEEK_SUNDAY')", + 'P1M': "nearestDate({col}, 'MONTH')", + 'P0.25Y': "nearestDate({col}, 'QUARTER')", + 'P1Y': "nearestDate({col}, 'YEAR')", + } + + # Returns a function to convert a Unix timestamp in milliseconds to a date + @classmethod + def epoch_to_dttm(cls): + return 'TO_DATE({col})' + + @classmethod + def convert_dttm(cls, target_type, dttm): + tt = target_type.upper() + if tt == 'DATE': + return "CAST('{}' AS DATE)".format(dttm.isoformat()[:10]) + elif tt == 'TIMESTAMP': + return "CAST('{}' AS TIMESTAMP)".format( + dttm.strftime('%Y-%m-%d %H:%M:%S')) + return "'{}'".format(dttm.strftime('%Y-%m-%d %H:%M:%S')) + + @classmethod + def adjust_database_uri(cls, uri, selected_schema): + database = uri.database + if '/' in uri.database: + database = uri.database.split('/')[0] + if selected_schema: + uri.database = database + '/' + selected_schema + return uri + + @classmethod + def extra_table_metadata(cls, database, table_name, schema_name): + return super().extra_table_metadata(database, table_name, schema_name) + + @classmethod + def select_star(cls, my_db, table_name, engine, schema=None, limit=100, + show_cols=False, indent=True, latest_partition=True, + cols=None): + + return super().select_star(my_db, table_name, engine, + schema, limit, + show_cols, indent, latest_partition, cols) + + @classmethod + def get_table_names(cls, inspector, schema): + return sorted(inspector.get_table_names(schema)) + + @classmethod + def get_view_names(cls, inspector, schema): + return sorted(inspector.get_view_names(schema)) + + class MySQLEngineSpec(BaseEngineSpec): engine = 'mysql' max_column_name_length = 64 From 55e1a72e661f2a60f21f778084d55f98385093ac Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Wed, 24 Apr 2019 00:33:04 -0400 Subject: [PATCH 2/7] Updated Docs --- docs/installation.rst | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index 3e2934a37dff1..3405b8ab2bb95 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -392,6 +392,12 @@ Here's a list of some of the recommended packages. | Pinot | ``pip install pinotdb`` | ``pinot+http://controller:5436/`` | | | | ``query?server=http://controller:5983/`` | +---------------+-------------------------------------+-------------------------------------------------+ +| Apache Drill | | For the REST API:`` | +| | | ``drill+sadrill://`` | +| | | For JDBC | +| | | ``drill+jdbc://`` | ++---------------+-------------------------------------+-------------------------------------------------+ + Note that many other databases are supported, the main criteria being the existence of a functional SqlAlchemy dialect and Python driver. Googling @@ -449,6 +455,31 @@ Required environment variables: :: See `Teradata SQLAlchemy `_. +Apache Drill +--------- +At the time of writing, the SQLAlchemy Dialect is not available on pypi and must be downloaded here: +`SQLAlchemy Drill `_ + +Alternatively, you can install it completely from the command line as follows: :: + + git clone https://github.com/JohnOmernik/sqlalchemy-drill + cd sqlalchemy-drill + python3 setup.py install + +Once that is done, you can connect to Drill in two ways, either via the REST interface or by JDBC. If you are connecting via JDBC, you must have the +Drill JDBC Driver installed. + +The basic connection string for Drill looks like this :: + + drill+sadrill://{username}:{password}@{host}:{port}/{storage_plugin}?use_ssl=True + +If you are using JDBC to connect to Drill, the connection string looks like this: :: + + drill+jdbc://{username}:{password}@{host}:{port}/{storage_plugin} + +For a complete tutorial about how to use Apache Drill with Superset, see this tutorial: +`Visualize Anything with Superset and Drill `_ + Caching ------- From 5bb3eaf25200ac5bcbc957614341a1f689a1cc23 Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Sun, 26 May 2019 13:35:28 -0400 Subject: [PATCH 3/7] Removed Extraneous Functions --- superset/db_engine_specs.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/superset/db_engine_specs.py b/superset/db_engine_specs.py index da99e878c1b41..c767a7536ef7b 100644 --- a/superset/db_engine_specs.py +++ b/superset/db_engine_specs.py @@ -766,10 +766,6 @@ def adjust_database_uri(cls, uri, selected_schema): uri.database = database + '/' + selected_schema return uri - @classmethod - def extra_table_metadata(cls, database, table_name, schema_name): - return super().extra_table_metadata(database, table_name, schema_name) - @classmethod def select_star(cls, my_db, table_name, engine, schema=None, limit=100, show_cols=False, indent=True, latest_partition=True, From 940f87ae6182a652b385eac2b5647e9566689933 Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Sun, 26 May 2019 13:39:05 -0400 Subject: [PATCH 4/7] Removed Extraneous Functions --- superset/db_engine_specs.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/superset/db_engine_specs.py b/superset/db_engine_specs.py index c767a7536ef7b..c5eb9390bd61e 100644 --- a/superset/db_engine_specs.py +++ b/superset/db_engine_specs.py @@ -775,14 +775,6 @@ def select_star(cls, my_db, table_name, engine, schema=None, limit=100, schema, limit, show_cols, indent, latest_partition, cols) - @classmethod - def get_table_names(cls, inspector, schema): - return sorted(inspector.get_table_names(schema)) - - @classmethod - def get_view_names(cls, inspector, schema): - return sorted(inspector.get_view_names(schema)) - class MySQLEngineSpec(BaseEngineSpec): engine = 'mysql' From 9b94ae50d0c596335b0880001e4dce142bb933ef Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Sun, 26 May 2019 15:48:29 -0400 Subject: [PATCH 5/7] Final Mods --- superset/db_engine_specs.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/superset/db_engine_specs.py b/superset/db_engine_specs.py index c5eb9390bd61e..79808c7bc5fc5 100644 --- a/superset/db_engine_specs.py +++ b/superset/db_engine_specs.py @@ -759,22 +759,12 @@ def convert_dttm(cls, target_type, dttm): @classmethod def adjust_database_uri(cls, uri, selected_schema): - database = uri.database if '/' in uri.database: database = uri.database.split('/')[0] if selected_schema: - uri.database = database + '/' + selected_schema + uri.database = parse.quote(selected_schema, safe='') return uri - @classmethod - def select_star(cls, my_db, table_name, engine, schema=None, limit=100, - show_cols=False, indent=True, latest_partition=True, - cols=None): - - return super().select_star(my_db, table_name, engine, - schema, limit, - show_cols, indent, latest_partition, cols) - class MySQLEngineSpec(BaseEngineSpec): engine = 'mysql' From 90671966f011376f339d5a1b9cce19a97ec4c5d1 Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Sun, 26 May 2019 16:08:39 -0400 Subject: [PATCH 6/7] Fixed Unit Test Error --- superset/db_engine_specs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/superset/db_engine_specs.py b/superset/db_engine_specs.py index 79808c7bc5fc5..01122b4ce90f9 100644 --- a/superset/db_engine_specs.py +++ b/superset/db_engine_specs.py @@ -759,8 +759,6 @@ def convert_dttm(cls, target_type, dttm): @classmethod def adjust_database_uri(cls, uri, selected_schema): - if '/' in uri.database: - database = uri.database.split('/')[0] if selected_schema: uri.database = parse.quote(selected_schema, safe='') return uri From 7209c9d85edc919f7b073facf825019a2c582cb7 Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Mon, 27 May 2019 22:23:36 -0400 Subject: [PATCH 7/7] Fixed Epoch Conversion Functions --- superset/db_engine_specs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/superset/db_engine_specs.py b/superset/db_engine_specs.py index 01122b4ce90f9..04efef78b8f37 100644 --- a/superset/db_engine_specs.py +++ b/superset/db_engine_specs.py @@ -745,6 +745,10 @@ class DrillEngineSpec(BaseEngineSpec): # Returns a function to convert a Unix timestamp in milliseconds to a date @classmethod def epoch_to_dttm(cls): + return cls.epoch_ms_to_dttm().replace('{col}', '({col}*1000)') + + @classmethod + def epoch_ms_to_dttm(cls): return 'TO_DATE({col})' @classmethod