Skip to content

Commit

Permalink
fix: fix redash dashboard exporter (#422)
Browse files Browse the repository at this point in the history 
* fix redash dashboard exporter

Signed-off-by: Lingkai Kong <lingkai.kong@databricks.com>

* fix mypy issue

Signed-off-by: Lingkai Kong <lingkai.kong@databricks.com>
  • Loading branch information
@gjxdxh
gjxdxh authored Dec 9, 2020
1 parent 4c9e5f7 commit fa626f5
Show file tree
Hide file tree
Showing 4 changed files with 85 additions and 16 deletions.
22 changes: 16 additions & 6 deletions databuilder/extractor/dashboard/redash/redash_dashboard_extractor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from databuilder.models.dashboard.dashboard_owner import DashboardOwner
from databuilder.models.dashboard.dashboard_query import DashboardQuery
from databuilder.models.dashboard.dashboard_table import DashboardTable
from databuilder.models.dashboard.dashboard_chart import DashboardChart
from databuilder.models.table_metadata import TableMetadata
from databuilder.extractor.base_extractor import Extractor
from databuilder.rest_api.rest_api_query import RestApiQuery
Expand Down Expand Up @@ -113,8 +114,8 @@ def _get_extract_iter(self) -> Iterator[Any]:
identity_data = {
'cluster': self._cluster,
'product': RedashDashboardExtractor.PRODUCT,
'dashboard_group_id': RedashDashboardExtractor.DASHBOARD_GROUP_ID,
'dashboard_id': record['dashboard_id']
'dashboard_group_id': str(RedashDashboardExtractor.DASHBOARD_GROUP_ID),
'dashboard_id': str(record['dashboard_id'])
}

dash_data = {
Expand All @@ -125,8 +126,8 @@ def _get_extract_iter(self) -> Iterator[Any]:
'dashboard_name':
record['dashboard_name'],
'dashboard_url':
'{redash}/dashboard/{slug}'
.format(redash=self._redash_base_url, slug=record['slug']),
'{redash}/dashboards/{id}'
.format(redash=self._redash_base_url, id=record['dashboard_id']),
'created_timestamp':
record['created_timestamp']
}
Expand Down Expand Up @@ -155,7 +156,7 @@ def _get_extract_iter(self) -> Iterator[Any]:

for viz in viz_widgets:
query_data = {
'query_id': viz.query_id,
'query_id': str(viz.query_id),
'query_name': viz.query_name,
'url': self._redash_base_url + viz.query_relative_url,
'query_text': viz.raw_query
Expand All @@ -164,6 +165,15 @@ def _get_extract_iter(self) -> Iterator[Any]:
query_data.update(identity_data)
yield DashboardQuery(**query_data)

chart_data = {
'query_id': str(viz.query_id),
'chart_id': str(viz.visualization_id),
'chart_name': viz.visualization_name,
'chart_type': viz.visualization_type,
}
chart_data.update(identity_data)
yield DashboardChart(**chart_data)

# if a table parser is provided, retrieve tables from this viz
if self._parse_tables:
for tbl in self._parse_tables(viz):
Expand Down Expand Up @@ -197,7 +207,7 @@ def _build_restapi_query(self) -> RestApiQuery:

return RestApiQuery(
query_to_join=dashes_query,
url='{redash_api}/dashboards/{{slug}}'.format(redash_api=self._api_base_url),
url='{redash_api}/dashboards/{{dashboard_id}}'.format(redash_api=self._api_base_url),
params=self._get_default_api_query_params(),
json_path='widgets',
field_names=['widgets'],
Expand Down
14 changes: 13 additions & 1 deletion databuilder/extractor/dashboard/redash/redash_dashboard_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,18 @@ def query_relative_url(self) -> str:
def query_name(self) -> str:
return self._data['visualization']['query']['name']

@property
def visualization_id(self) -> int:
return self._data['visualization']['id']

@property
def visualization_name(self) -> str:
return self._data['visualization']['name']

@property
def visualization_type(self) -> str:
return self._data['visualization']['type']


class RedashTextWidget:
"""
Expand Down Expand Up @@ -135,7 +147,7 @@ def generate_dashboard_description(text_widgets: List[RedashTextWidget],
if len(text_widgets) > 0:
return '\n\n'.join([w.text for w in text_widgets])
elif len(viz_widgets) > 0:
query_list = '\n'.join(['- {}'.format(v.query_name) for v in set(viz_widgets)])
query_list = '\n'.join(set(['- {}'.format(v.query_name) for v in set(viz_widgets)]))
return 'A dashboard containing the following queries:\n\n' + query_list

return 'This dashboard appears to be empty!'
31 changes: 23 additions & 8 deletions tests/unit/extractor/dashboard/redash/test_redash_dashboard_extractor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from databuilder.models.dashboard.dashboard_owner import DashboardOwner
from databuilder.models.dashboard.dashboard_query import DashboardQuery
from databuilder.models.dashboard.dashboard_table import DashboardTable
from databuilder.models.dashboard.dashboard_chart import DashboardChart


logging.basicConfig(level=logging.INFO)
Expand Down Expand Up @@ -43,18 +44,21 @@ def test_table_relation_data(self) -> None:

def test_with_one_dashboard(self) -> None:
def mock_api_get(url: str, *args: Any, **kwargs: Any) -> MockApiResponse:
if 'test-dash' in url:
if '1000' in url:
return MockApiResponse({
'id': 123,
'id': 1000,
'widgets': [
{
'visualization': {
'query': {
'data_source_id': 1,
'id': '1234',
'id': 1234,
'name': 'Test Query',
'query': 'SELECT id FROM users'
}
},
'id': 12345,
'name': 'test_widget',
'type': 'CHART',
},
'options': {}
}
Expand All @@ -67,7 +71,7 @@ def mock_api_get(url: str, *args: Any, **kwargs: Any) -> MockApiResponse:
'page_size': 50,
'results': [
{
'id': 123,
'id': 1000,
'name': 'Test Dash',
'slug': 'test-dash',
'created_at': '2020-01-01T00:00:00.000Z',
Expand Down Expand Up @@ -96,20 +100,20 @@ def mock_api_get(url: str, *args: Any, **kwargs: Any) -> MockApiResponse:

# DashboardMetadata
record = extractor.extract()
self.assertEqual(record.dashboard_id, 123)
self.assertEqual(record.dashboard_id, '1000')
self.assertEqual(record.dashboard_name, 'Test Dash')
self.assertEqual(record.dashboard_group_id, RedashDashboardExtractor.DASHBOARD_GROUP_ID)
self.assertEqual(record.dashboard_group, RedashDashboardExtractor.DASHBOARD_GROUP_NAME)
self.assertEqual(record.product, RedashDashboardExtractor.PRODUCT)
self.assertEqual(record.cluster, RedashDashboardExtractor.DEFAULT_CLUSTER)
self.assertEqual(record.created_timestamp, 1577836800)
self.assertTrue(redash_base_url in record.dashboard_url)
self.assertTrue('test-dash' in record.dashboard_url)
self.assertTrue('1000' in record.dashboard_url)

# DashboardLastModified
record = extractor.extract()
identity: Dict[str, Any] = {
'dashboard_id': 123,
'dashboard_id': '1000',
'dashboard_group_id': RedashDashboardExtractor.DASHBOARD_GROUP_ID,
'product': RedashDashboardExtractor.PRODUCT,
'cluster': u'prod'
Expand All @@ -136,6 +140,17 @@ def mock_api_get(url: str, *args: Any, **kwargs: Any) -> MockApiResponse:
)
self.assertEqual(record.__repr__(), expected_query.__repr__())

# DashboardChart
record = extractor.extract()
expected_chart = DashboardChart(
query_id='1234',
chart_id='12345',
chart_name='test_widget',
chart_type='CHART',
**identity
)
self.assertEqual(record.__repr__(), expected_chart.__repr__())

# DashboardTable
record = extractor.extract()
expected_table = DashboardTable(
Expand Down
34 changes: 33 additions & 1 deletion tests/unit/extractor/dashboard/redash/test_redash_dashboard_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,10 @@ def test_visualization_widget_props(self) -> None:
'data_source_id': 1,
'query': 'SELECT 2+2 FROM DUAL',
'name': 'Test'
}
},
'id': 12345,
'name': 'test_widget',
'type': 'CHART'
}
}
widget = get_visualization_widgets([widget_data])[0]
Expand All @@ -75,6 +78,9 @@ def test_visualization_widget_props(self) -> None:
self.assertEqual(widget.data_source_id, 1)
self.assertEqual(widget.raw_query, 'SELECT 2+2 FROM DUAL')
self.assertEqual(widget.query_name, 'Test')
self.assertEqual(widget.visualization_id, 12345)
self.assertEqual(widget.visualization_name, 'test_widget')
self.assertEqual(widget.visualization_type, 'CHART')

def test_descriptions_from_text(self) -> None:
text_widgets = get_text_widgets([
Expand Down Expand Up @@ -123,6 +129,32 @@ def test_descriptions_from_text(self) -> None:
desc4 = generate_dashboard_description([], [])
self.assertTrue('empty' in desc4)

def test_descriptions_remove_duplicate(self) -> None:
viz_widgets = get_visualization_widgets([
{
'visualization': {
'query': {
'id': 1,
'data_source_id': 1,
'name': 'same_query_name',
'query': 'n/a'
}
}
},
{
'visualization': {
'query': {
'id': 2,
'data_source_id': 1,
'name': 'same_query_name',
'query': 'n/a'
}
}
}
])
desc1 = generate_dashboard_description([], viz_widgets)
self.assertEqual('A dashboard containing the following queries:\n\n- same_query_name', desc1)

def test_auth_headers(self) -> None:
headers = get_auth_headers('testkey')
self.assertTrue('testkey' in headers['Authorization'])
Expand Down

0 comments on commit fa626f5

Please sign in to comment.