From ef9a4d5eed82605ad672b4717c75c87c4b8609fa Mon Sep 17 00:00:00 2001 From: Gleb Lesnikov Date: Mon, 26 Aug 2019 10:17:49 +0300 Subject: [PATCH] [Data Sources] Add: Azure Data Explorer (Kusto) query runner (#4091) * [Data Sources] Add: Azure Data Explorer (Kusto) query runner * CodeClimate fixes * Remove TODO * Fixed configuration properties names for Azure Kusto * Azure Kusto: get_schema in one query * azure-kusto-data update to 0.0.32 * Add Kusto to the default query runners list --- .../assets/images/db-logos/azure_kusto.png | Bin 0 -> 5070 bytes redash/query_runner/azure_kusto.py | 159 ++++++++++++++++++ redash/settings/__init__.py | 1 + requirements.txt | 2 +- requirements_all_ds.txt | 1 + 5 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 client/app/assets/images/db-logos/azure_kusto.png create mode 100644 redash/query_runner/azure_kusto.py diff --git a/client/app/assets/images/db-logos/azure_kusto.png b/client/app/assets/images/db-logos/azure_kusto.png new file mode 100644 index 0000000000000000000000000000000000000000..9f24192c3eb1538d220f6f5317473e3c0bf399d2 GIT binary patch literal 5070 zcmeHL`9GB1`@d%zHKPTi#ZnALWKBp3Glk3`WSi^?dCFEHTbY@XA~Duv8;^t%$`zo)X3zNNq5`q8# z?8W1ZtN{Q5HX#6fC-~=ltLQ2Kh^FF=&f12${>r`^ccd@u;Y95RpUkSd*Plpr5Xm<$ zk+(k$HCqi+gpiLgR(*;wc^pGW+;emZO7Y^WZY=C2zF5f=l`G8ZQ+j#)rRj^Jy{-gd zmW;K|{J8eoFsxZareIKQuti!dbF9v$GsCC9WBb(HmuJrcXZ6n%Y{68Z0j{oAEabB?RgMOHm|WwG&f@i?hO4qN*aP9%@TEBN@Pky0w9cCl-xze zqbflGSU0!pG71UT;D26u2zw%L1_{^ZcZv|XO-T6ZKh52_QAoHxzxe}^YlLi-}jlP`5TLHld=1ptb?#Reepay!JkWq+iRpwA77LrqY8{Brej71nNNe%hgK?|l# z0C>GKAmxhKe}t=orX1zN$QLDO#Ft9 zkZPLpA^N~&57~ksxXC4PQeLZMd z0eOe{9kKDt>~qTO$Q}$73z%kF_)z5d7!jG^beZ8D^p#+G>oRL1BEB-){|sMgIvv!5?-{oT>$)_FcPCFn^?2Zojec zDNTOzI(spTTNLo|PSB+9{L%vD`M=4m>=K>=uj)gMWaN|0&jVx|#`S%<#`Q@sXCY!X zp4C6;Fy_K)=(#CgR>hULVIQzswIBrT9y+Ege&_kj0r@lg4ku42Mw$=or!5c|KSMs=><5O5#Xo_(Q^dyK5tK-FuT zEBN&BrAb(y8}Dr(O^^NU@%*CS4Ol9(tTvZW8g4^x8}d+>N+pJE8b7OKy34>Qc&F1v zCm_a%nFHLbmaHM>7sv*8?NCaGwRjPdyQt=jB95-N1c)0{@rqs(mFi%^B-28%WL5G> zq@pfob=uDqb%DLO_*Mo2qx|Zh{qYKxfvZntj--&DFh`OAsMT0*Tt`m*9z@N-NJ-=y zaqxi!h5~%Lat$Z75i=rOPltET)t5>y<@3-~>7_vhOlUaO!YdESU03^+8j`cVxfjzX ziq_dVz;ooCmxo%(f2kRT+xm~O9)zsY-!EsyQD*|%8vJWlh3ML^ls7IhR1+|A>K{-1 zt>q@q*?l#n)!TjzpZ_~`dR~Fo)n8Y>JbClQ>bG4ofU5JJuE+JSr1iEtHD}vDm+MVb znOQ!S+}WjJ(ER?w2hXjKgyAj%f_RJK1$uT1xne}BY={IxRHxa7mUpjg!0W|yL@SO` zs$gA=XR0ACRkr5bMtuW^lNV)ca&Wx;yx=C9o=OJgjvlVeuh*$ z2UVJKyBk^N9PFQfrY@REjVE&iyIUseQjYB&yDta0T!{-w!sZ~Y5X1_F4K!^ByH#^n z*5yY))-{3Mcr%m`A|FAlGkA(70biGNXMIFNCy8o{y1dFZq2`+}w%BIsn3Y0;{@AAB zs)YBY?d~#KoxAJG#V5zw4gI1f#)2`llG=jX|bT+#+7r46Dm}fu&TMs zG2h>d&L!w1NEbVADJYw)P_OpnlT=D!zR?o`ps027LRSWdRhA~ePe^np0MCBzvCLDA z&8yPv+MZ#*{UJe+&qZ=)q9;b^eZ(X>J?D&~B++}#SgM!X*NAZt6|Y4=(r?9wY zDwA%;BZ{JUKbO>M!*0Co2^2_T{GPD6>*{KCr<}5r6>V7f5F-w*6X_M{1QnbLu?D%b zJeZ^B__HyXOMeV?GK&$<=!jruF3g@ zp0wugkT+@S#Z!kimbIVS@`i?|GY%^z)<5tLiuC=I)n#t6TDlyLu2rY#O`oY9;SDl` z+ARhDnHng<8SJ9q7aMhp5~j*RezZvIFOM1Pr%IMD42lQp7Us+(bKF0Z?~O0`*hxZL zUoDdl56m&=!#sOY? zXbDK1B!8)u0#>ukEi^=uKZG>{)S`KbB|AEFuc!9fmkgdLurts6cT~_l=1dvjm2}m+Ob_@$(C~q$MHTwoX_lV6JGr zYs=RgKLQpWV2;l!Z+6+kV^;qijO>`R5vW1BRBuivUVr8BB|Yl}j3vmARSNHduQjS8 zL(c!M*WP|U;wOkYZGRBis>zo(_iAWn7QX*UOZ_G3Z4aEXgFqDLlN4!CTJ6)j!~~ZACnUwS-2|N|c>I|Nc|R zSonK>SB5l9oaB@&{cmkzG4#bx-fEf|aEJAk98;wHe(7ssNjridc52(SzKTB&Gg@`j z7An7&c-U-(IugC~*qF=G7MmDeDeJQwz4`FlepZPVItr;wH}pQN{V@0Kr>%b#H`&Kr;=fDl=sR!k&P0m65BdBamlrf~t<_ywmb51~ZHkiM z>o{)nY+zYFk=>w}6hT-`@W(klH4E7)>YObKD3@Z5ifl>aFQX;zOldROT3=t;n;JaIW- z)PNlv^V~B1wL2nJd2MIcT!TIY=3XwD-kXMf)<7&D#Hl%sxsCNyhdzNAn*or& zv>yL`9cC)QZ5wWC+VCOxhFLthw^<}BPXfMeHQ@xHFZR$LHC3_uwV%dw>!8r*GR7b< zv8u=TC-9ns+%(3rPv!A6_L}yN=ZjheVtZB3LE-g9cIFv)5_4ooGz?TRIPcVLBEz{o zLM;Sqq_J^<5UmF4g^p#g25K!Jo;{q(^d6bjLz95=Uze{XSg@C``SpXP(8kJJTW=Z8 zMzy9Hco@ACF~2nEy)Oyq$35 zyI~I-5J}0n?mLMFhO=40K~FQV-^tZ|tJJ_SlFU$Rhz8M&=&Sh#N70V2I4W2uLtwcN z&cuC+nkX|U1Q}-*4?HVNV|vG*N)}Bw1t6YZ_hE!g+1o#z?)Du8HASwE4lf{R+9?mD za0C!k_wCx7w;CAdrF~1znSi234je|<+!t_a;! zp}ZsK{|t zc~?@GA5O^dNV!}Z3!bgTb+hxLK}YD6?GfxXY_i^3$RKx8PiA8R$3danB4Ll`f5Uk% zXNEzer7u+`eHnQZ{KGnCMkBcx7@4x9>`T~NDjXwq;C?Z(VItI~8nlTbp3_uD<8q4lGX)T990ou3E z##tv~rKZXn?Hw&GFvAzq_8tfAkgJkiWouI0{aqY9Krk^hhu%WaG14_uR$rgz2bL0W zieA4@@4*%6#=pqEdk)HSH};G%>Um07B`{RV2WQ4&6_bd_n5{hAl4BnxJRouLSYz!( zI6Xc%6H~>hKJo2}qoJp6Vwfc>yG{{Axr0|5V$VHfVH^b#z+uQ7z}AB|7{||lHiEM)0SJ4-P>Dp+0Jl(Fe2g%Y z3VH?Yp)COM0PQPwL{w%H1A$EvgHKz Z9E@>Z;G@%2HxB^(;f*bfO3%69`w!&6*HHif literal 0 HcmV?d00001 diff --git a/redash/query_runner/azure_kusto.py b/redash/query_runner/azure_kusto.py new file mode 100644 index 0000000000..a2eb32954d --- /dev/null +++ b/redash/query_runner/azure_kusto.py @@ -0,0 +1,159 @@ +from redash.query_runner import BaseQueryRunner, register +from redash.query_runner import TYPE_STRING, TYPE_DATE, TYPE_DATETIME, TYPE_INTEGER, TYPE_FLOAT, TYPE_BOOLEAN +from redash.utils import json_dumps, json_loads + + +try: + from azure.kusto.data.request import KustoClient, KustoConnectionStringBuilder + from azure.kusto.data.exceptions import KustoServiceError + enabled = True +except ImportError: + enabled = False + +TYPES_MAP = { + 'boolean': TYPE_BOOLEAN, + 'datetime': TYPE_DATETIME, + 'date': TYPE_DATE, + 'dynamic': TYPE_STRING, + 'guid': TYPE_STRING, + 'int': TYPE_INTEGER, + 'long': TYPE_INTEGER, + 'real': TYPE_FLOAT, + 'string': TYPE_STRING, + 'timespan': TYPE_STRING, + 'decimal': TYPE_FLOAT +} + + +class AzureKusto(BaseQueryRunner): + noop_query = "let noop = datatable (Noop:string)[1]; noop" + + def __init__(self, configuration): + super(AzureKusto, self).__init__(configuration) + self.syntax = 'custom' + + @classmethod + def configuration_schema(cls): + return { + "type": "object", + "properties": { + "cluster": { + "type": "string" + }, + "azure_ad_client_id": { + "type": "string", + "title": "Azure AD Client ID" + }, + "azure_ad_client_secret": { + "type": "string", + "title": "Azure AD Client Secret" + }, + "azure_ad_tenant_id": { + "type": "string", + "title": "Azure AD Tenant Id" + }, + "database": { + "type": "string" + } + }, + "required": [ + "cluster", "azure_ad_client_id", "azure_ad_client_secret", + "azure_ad_tenant_id", "database" + ], + "order": [ + "cluster", "azure_ad_client_id", "azure_ad_client_secret", + "azure_ad_tenant_id", "database" + ], + "secret": ["azure_ad_client_secret"] + } + + @classmethod + def enabled(cls): + return enabled + + @classmethod + def annotate_query(cls): + return False + + @classmethod + def type(cls): + return "azure_kusto" + + @classmethod + def name(cls): + return "Azure Data Explorer (Kusto)" + + def run_query(self, query, user): + + kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( + connection_string=self.configuration['cluster'], + aad_app_id=self.configuration['azure_ad_client_id'], + app_key=self.configuration['azure_ad_client_secret'], + authority_id=self.configuration['azure_ad_tenant_id']) + + client = KustoClient(kcsb) + + db = self.configuration['database'] + try: + response = client.execute(db, query) + + result_cols = response.primary_results[0].columns + result_rows = response.primary_results[0].rows + + columns = [] + rows = [] + for c in result_cols: + columns.append({ + 'name': c.column_name, + 'friendly_name': c.column_name, + 'type': TYPES_MAP.get(c.column_type, None) + }) + + # rows must be [{'column1': value, 'column2': value}] + for row in result_rows: + rows.append(row.to_dict()) + + error = None + data = {'columns': columns, 'rows': rows} + json_data = json_dumps(data) + + except KustoServiceError as err: + json_data = None + try: + error = err.args[1][0]['error']['@message'] + except (IndexError, KeyError): + error = err.args[1] + except KeyboardInterrupt: + json_data = None + error = "Query cancelled by user." + + return json_data, error + + def get_schema(self, get_stats=False): + query = ".show database schema as json" + + results, error = self.run_query(query, None) + + if error is not None: + raise Exception("Failed getting schema.") + + results = json_loads(results) + + schema_as_json = json_loads(results['rows'][0]['DatabaseSchema']) + tables_list = schema_as_json['Databases'][self.configuration['database']]['Tables'].values() + + schema = {} + + for table in tables_list: + table_name = table['Name'] + + if table_name not in schema: + schema[table_name] = {'name': table_name, 'columns': []} + + for column in table['OrderedColumns']: + schema[table_name]['columns'].append(column['Name']) + + return schema.values() + + +register(AzureKusto) diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index e2f0323043..1a7154dfe6 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -293,6 +293,7 @@ def email_server_is_configured(): 'redash.query_runner.json_ds', 'redash.query_runner.cass', 'redash.query_runner.dgraph', + 'redash.query_runner.azure_kusto', ] enabled_query_runners = array_from_string(os.environ.get("REDASH_ENABLED_QUERY_RUNNERS", ",".join(default_query_runners))) diff --git a/requirements.txt b/requirements.txt index 6a3ff84c21..b7e2b5e9c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ passlib==1.6.2 aniso8601==1.1.0 blinker==1.3 psycopg2==2.7.3.2 -python-dateutil==2.7.5 +python-dateutil==2.8.0 pytz==2016.7 PyYAML==3.12 redis==3.2.1 diff --git a/requirements_all_ds.txt b/requirements_all_ds.txt index 387b45b3e4..5a0b67898d 100644 --- a/requirements_all_ds.txt +++ b/requirements_all_ds.txt @@ -31,3 +31,4 @@ phoenixdb==0.7 # certifi is needed to support MongoDB and SSL: certifi pydgraph==1.2.0 +azure-kusto-data==0.0.32