From 41247074f4986c0855ea0f7db4ad0921fc5d9abe Mon Sep 17 00:00:00 2001 From: pl Date: Wed, 5 Apr 2017 15:52:04 +0200 Subject: [PATCH 1/7] sql_lab.py: compress via utils --- superset/sql_lab.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/superset/sql_lab.py b/superset/sql_lab.py index cc42ed96fb389..f3dcbf120b65b 100644 --- a/superset/sql_lab.py +++ b/superset/sql_lab.py @@ -6,7 +6,6 @@ import pandas as pd import sqlalchemy import uuid -import zlib from sqlalchemy.pool import NullPool from sqlalchemy.orm import sessionmaker @@ -185,7 +184,7 @@ def handle_error(msg): if store_results: key = '{}'.format(uuid.uuid4()) logging.info("Storing results in results backend, key: {}".format(key)) - results_backend.set(key, zlib.compress(payload)) + results_backend.set(key, utils.zlib_compress(payload)) query.results_key = key session.merge(query) From 0e70ce51144133c9368a4ef6625c6c3a072ab081 Mon Sep 17 00:00:00 2001 From: pl Date: Wed, 5 Apr 2017 16:01:48 +0200 Subject: [PATCH 2/7] utils.py: added zlib_compress and zlib_compress_to_string --- superset/utils.py | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/superset/utils.py b/superset/utils.py index 984f9c6834049..7081458f3d30d 100644 --- a/superset/utils.py +++ b/superset/utils.py @@ -16,6 +16,8 @@ import sqlalchemy as sa import signal import uuid +import sys +import zlib from builtins import object from datetime import date, datetime, time @@ -41,7 +43,7 @@ logging.getLogger('MARKDOWN').setLevel(logging.INFO) - +PY3K = sys.version_info >= (3, 0) EPOCH = datetime(1970, 1, 1) DTTM_ALIAS = '__timestamp' @@ -572,3 +574,46 @@ def setup_cache(app, cache_config): """Setup the flask-cache on a flask app""" if cache_config and cache_config.get('CACHE_TYPE') != 'null': return Cache(app, config=cache_config) + + + +def zlib_compress(data): + """ + compress things in a py2/3 safe fashion + + >>> json_str = '{"test": 1}' + >>> blob = zlib_compress(json_str) + """ + + if PY3K: + if isinstance(data, str): + return zlib.compress(bytes(data, "utf-8")) + else: + return zlib.compress(data) + else: + return zlib.compress(data) + + +def zlib_uncompress_to_string(blob): + """ + uncompress things to a string in a py2/3 safe fashion + >>> json_str = '{"test": 1}' + >>> blob = zlib_compress(json_str) + >>> got_str = zlib_uncompress_to_string(blob) + >>> got_str == json_str + True + """ + + if PY3K: + decompressed = "" + if isinstance(blob, bytes): + decompressed = zlib.decompress(blob) + else: + decompressed = zlib.decompress(bytes(blob, "utf-8")) + + if isinstance(decompressed, str): + return decompressed + return decompressed.decode("utf-8") + else: + return zlib.decompress(blob) + From 0b450d4047d4ddbe33d55389df7cff556ce0e6b8 Mon Sep 17 00:00:00 2001 From: pl Date: Wed, 5 Apr 2017 16:14:37 +0200 Subject: [PATCH 3/7] core.py: converted to use zlib_decompress_to_string; renamed uncompress to decompress in utils.py --- superset/utils.py | 2 +- superset/views/core.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/superset/utils.py b/superset/utils.py index 7081458f3d30d..239fc237ebb14 100644 --- a/superset/utils.py +++ b/superset/utils.py @@ -594,7 +594,7 @@ def zlib_compress(data): return zlib.compress(data) -def zlib_uncompress_to_string(blob): +def zlib_decompress_to_string(blob): """ uncompress things to a string in a py2/3 safe fashion >>> json_str = '{"test": 1}' diff --git a/superset/views/core.py b/superset/views/core.py index 5f917c82c3f57..f85ecffaa25c1 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -11,7 +11,6 @@ import re import time import traceback -import zlib import sqlalchemy as sqla @@ -1886,7 +1885,7 @@ def results(self, key): return json_error_response(get_datasource_access_error_msg( '{}'.format(rejected_tables))) - payload = zlib.decompress(blob) + payload = utils.zlib_decompress_to_string(blob) display_limit = app.config.get('DISPLAY_SQL_MAX_ROW', None) if display_limit: payload_json = json.loads(payload) @@ -2026,7 +2025,7 @@ def csv(self, client_id): if results_backend and query.results_key: blob = results_backend.get(query.results_key) if blob: - json_payload = zlib.decompress(blob) + json_payload = utils.zlib_decompress_to_string(blob) obj = json.loads(json_payload) columns = [c['name'] for c in obj['columns']] df = pd.DataFrame.from_records(obj['data'], columns=columns) From a11bcf6494609710f37bcedf35b42e165a0f10c9 Mon Sep 17 00:00:00 2001 From: pl Date: Wed, 5 Apr 2017 16:18:21 +0200 Subject: [PATCH 4/7] utils_tests.py: added test for compress/decompress --- tests/utils_tests.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/utils_tests.py b/tests/utils_tests.py index 231f03b84fe8a..5ebe48767a4d8 100644 --- a/tests/utils_tests.py +++ b/tests/utils_tests.py @@ -1,7 +1,7 @@ from datetime import datetime, date, timedelta, time from decimal import Decimal from superset.utils import ( - json_int_dttm_ser, json_iso_dttm_ser, base_json_conv, parse_human_timedelta + json_int_dttm_ser, json_iso_dttm_ser, base_json_conv, parse_human_timedelta, zlib_compress, zlib_decompress_to_string ) import unittest import uuid @@ -45,3 +45,10 @@ def test_base_json_conv(self): def test_parse_human_timedelta(self, mock_now): mock_now.return_value = datetime(2016, 12, 1) self.assertEquals(parse_human_timedelta('now'), timedelta(0)) + + def test_zlib_compression(self): + json_str = """{"test": 1}""" + blob = zlib_compress(json_str) + got_str = zlib_uncompress_to_string(blob) + self.assertEquals(json_str, got_str) + From 3bc0b61ea56ebe60907f4cb902a76f4603376368 Mon Sep 17 00:00:00 2001 From: pl Date: Thu, 6 Apr 2017 07:03:45 +0200 Subject: [PATCH 5/7] fixed broken utils test; removed redundant code and empty lines from utils.py --- superset/utils.py | 12 +----------- tests/utils_tests.py | 2 +- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/superset/utils.py b/superset/utils.py index 239fc237ebb14..315f2688cb840 100644 --- a/superset/utils.py +++ b/superset/utils.py @@ -576,15 +576,12 @@ def setup_cache(app, cache_config): return Cache(app, config=cache_config) - def zlib_compress(data): """ compress things in a py2/3 safe fashion - >>> json_str = '{"test": 1}' >>> blob = zlib_compress(json_str) """ - if PY3K: if isinstance(data, str): return zlib.compress(bytes(data, "utf-8")) @@ -603,17 +600,10 @@ def zlib_decompress_to_string(blob): >>> got_str == json_str True """ - if PY3K: - decompressed = "" if isinstance(blob, bytes): decompressed = zlib.decompress(blob) else: decompressed = zlib.decompress(bytes(blob, "utf-8")) - - if isinstance(decompressed, str): - return decompressed return decompressed.decode("utf-8") - else: - return zlib.decompress(blob) - + return zlib.decompress(blob) diff --git a/tests/utils_tests.py b/tests/utils_tests.py index 5ebe48767a4d8..e07d9594b3979 100644 --- a/tests/utils_tests.py +++ b/tests/utils_tests.py @@ -49,6 +49,6 @@ def test_parse_human_timedelta(self, mock_now): def test_zlib_compression(self): json_str = """{"test": 1}""" blob = zlib_compress(json_str) - got_str = zlib_uncompress_to_string(blob) + got_str = zlib_decompress_to_string(blob) self.assertEquals(json_str, got_str) From 9752509552561e6827f79c79d596fb9742b637df Mon Sep 17 00:00:00 2001 From: pl Date: Thu, 6 Apr 2017 08:44:36 +0200 Subject: [PATCH 6/7] utils.py: corrected docstrings, removed unnecessary 'else' --- superset/utils.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/superset/utils.py b/superset/utils.py index 315f2688cb840..655e71d67aff8 100644 --- a/superset/utils.py +++ b/superset/utils.py @@ -578,7 +578,7 @@ def setup_cache(app, cache_config): def zlib_compress(data): """ - compress things in a py2/3 safe fashion + Compress things in a py2/3 safe fashion >>> json_str = '{"test": 1}' >>> blob = zlib_compress(json_str) """ @@ -587,16 +587,15 @@ def zlib_compress(data): return zlib.compress(bytes(data, "utf-8")) else: return zlib.compress(data) - else: - return zlib.compress(data) + return zlib.compress(data) def zlib_decompress_to_string(blob): """ - uncompress things to a string in a py2/3 safe fashion + Decompress things to a string in a py2/3 safe fashion >>> json_str = '{"test": 1}' >>> blob = zlib_compress(json_str) - >>> got_str = zlib_uncompress_to_string(blob) + >>> got_str = zlib_decompress_to_string(blob) >>> got_str == json_str True """ From 248cace7318a86149a0dcaceee2d23ebf56cf9cd Mon Sep 17 00:00:00 2001 From: pl Date: Thu, 6 Apr 2017 17:55:35 +0200 Subject: [PATCH 7/7] removed yet another superfluous else --- superset/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/superset/utils.py b/superset/utils.py index 655e71d67aff8..ec4c18d89681c 100644 --- a/superset/utils.py +++ b/superset/utils.py @@ -585,8 +585,7 @@ def zlib_compress(data): if PY3K: if isinstance(data, str): return zlib.compress(bytes(data, "utf-8")) - else: - return zlib.compress(data) + return zlib.compress(data) return zlib.compress(data)