From affa48e5a95a9a8ba77727f8ee3cd992d54913cd Mon Sep 17 00:00:00 2001 From: "Michael S. Molina" Date: Mon, 28 Aug 2023 11:17:28 -0300 Subject: [PATCH 1/3] fix: Bumps Flask Caching to fix RCE vulnerability --- requirements/base.txt | 8 +++++--- setup.py | 4 ++-- tests/integration_tests/cachekeys/api_tests.py | 5 +++++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index f8a7728cbea4a..d54bd6e8a976a 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -31,8 +31,10 @@ bottleneck==1.3.7 # via pandas brotli==1.0.9 # via flask-compress -cachelib==0.4.1 - # via apache-superset +cachelib==0.6.0 + # via + # apache-superset + # flask-caching celery==5.2.2 # via apache-superset certifi==2023.7.22 @@ -100,7 +102,7 @@ flask-appbuilder==4.3.6 # via apache-superset flask-babel==1.0.0 # via flask-appbuilder -flask-caching==1.10.1 +flask-caching==1.11.1 # via apache-superset flask-compress==1.13 # via apache-superset diff --git a/setup.py b/setup.py index 75e03fd4b10f2..bafeda5911097 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ def get_git_sha() -> str: }, install_requires=[ "backoff>=1.8.0", - "cachelib>=0.4.1,<0.5", + "cachelib>=0.4.1,<0.7", "celery>=5.2.2, <6.0.0", "click>=8.0.3", "click-option-group", @@ -86,7 +86,7 @@ def get_git_sha() -> str: "deprecation>=2.1.0, <2.2.0", "flask>=2.2.5, <3.0.0", "flask-appbuilder>=4.3.6, <5.0.0", - "flask-caching>=1.10.1, <2.0", + "flask-caching>=1.11.1, <2.0", "flask-compress>=1.13, <2.0", "flask-talisman>=1.0.0, <2.0", "flask-login>=0.6.0, < 1.0", diff --git a/tests/integration_tests/cachekeys/api_tests.py b/tests/integration_tests/cachekeys/api_tests.py index c867ce7f5135a..db2c51d124635 100644 --- a/tests/integration_tests/cachekeys/api_tests.py +++ b/tests/integration_tests/cachekeys/api_tests.py @@ -27,6 +27,10 @@ SupersetTestCase, post_assert_metric, ) +from tests.integration_tests.fixtures.birth_names_dashboard import ( + load_birth_names_dashboard_with_slices, + load_birth_names_data, +) @pytest.fixture @@ -95,6 +99,7 @@ def test_invalidate_cache_bad_request(invalidate): assert rv.status_code == 400 +@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices") def test_invalidate_existing_caches(invalidate): schema = get_example_default_schema() or "" bn = SupersetTestCase.get_birth_names_dataset() From b3e9a9f49d17bcf2bdc1398a1de4f75092ad3eb1 Mon Sep 17 00:00:00 2001 From: "Michael S. Molina" Date: Wed, 30 Aug 2023 15:28:19 -0300 Subject: [PATCH 2/3] Removes cachelib explicit dependency --- CONTRIBUTING.md | 88 +++++++++---------- docker/pythonpath_dev/superset_config.py | 2 +- .../installation/async-queries-celery.mdx | 4 +- helm/superset/templates/_helpers.tpl | 2 +- requirements/base.txt | 4 +- setup.cfg | 2 +- setup.py | 1 - superset/config.py | 2 +- superset/extensions/__init__.py | 2 +- tests/integration_tests/core_tests.py | 1 + 10 files changed, 52 insertions(+), 56 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 91f224bc03cf7..94573f2a9e3f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -578,6 +578,7 @@ npm ci ``` Note that Superset uses [Scarf](https://docs.scarf.sh) to capture telemetry/analytics about versions being installed, including the `scarf-js` npm package. As noted elsewhere in this documentation, Scarf gathers aggregated stats for the sake of security/release strategy, and does not capture/retain PII. [You can read here](https://docs.scarf.sh/package-analytics/) about the package, and various means to opt out of it, but one easy way to opt out is to add this setting in `superset-frontent/package.json`: + ```json // your-package/package.json { @@ -598,10 +599,13 @@ There are three types of assets you can build: 3. `npm run build-instrumented`: instrumented application code for collecting code coverage from Cypress tests If this type of error comes while building assets(i.e using above commands): + ```bash Error: You must provide the URL of lib/mappings.wasm by calling SourceMapConsumer.initialize ``` + Then put this: + ```bash export NODE_OPTIONS=--no-experimental-fetch ``` @@ -925,28 +929,22 @@ For debugging locally using VSCode, you can configure a launch configuration fil ```json { - "version": "0.2.0", - "configurations": [ - { - "name": "Python: Flask", - "type": "python", - "request": "launch", - "module": "flask", - "env": { - "FLASK_APP": "superset", - "SUPERSET_ENV": "development" - }, - "args": [ - "run", - "-p 8088", - "--with-threads", - "--reload", - "--debugger" - ], - "jinja": true, - "justMyCode": true - } - ] + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Flask", + "type": "python", + "request": "launch", + "module": "flask", + "env": { + "FLASK_APP": "superset", + "SUPERSET_ENV": "development" + }, + "args": ["run", "-p 8088", "--with-threads", "--reload", "--debugger"], + "jinja": true, + "justMyCode": true + } + ] } ``` @@ -1031,24 +1029,24 @@ You are now ready to attach a debugger to the process. Using VSCode you can conf ```json { - "version": "0.2.0", - "configurations": [ + "version": "0.2.0", + "configurations": [ + { + "name": "Attach to Superset App in Docker Container", + "type": "python", + "request": "attach", + "connect": { + "host": "127.0.0.1", + "port": 5678 + }, + "pathMappings": [ { - "name": "Attach to Superset App in Docker Container", - "type": "python", - "request": "attach", - "connect": { - "host": "127.0.0.1", - "port": 5678 - }, - "pathMappings": [ - { - "localRoot": "${workspaceFolder}", - "remoteRoot": "/app" - } - ] - }, - ] + "localRoot": "${workspaceFolder}", + "remoteRoot": "/app" + } + ] + } + ] } ``` @@ -1349,7 +1347,7 @@ To do this, you'll need to: but perfect for testing (stores cache in `/tmp`) ```python - from cachelib.file import FileSystemCache + from flask_caching.backends.filesystemcache import FileSystemCache RESULTS_BACKEND = FileSystemCache('/tmp/sqllab') ``` @@ -1415,11 +1413,11 @@ Note not all fields are correctly categorized. The fields vary based on visualiz ### Time -| Field | Type | Notes | -| ------------------ | -------- | ------------------------------------- | -| `granularity_sqla` | _string_ | The SQLA **Time Column** widget | -| `time_grain_sqla` | _string_ | The SQLA **Time Grain** widget | -| `time_range` | _string_ | The **Time range** widget | +| Field | Type | Notes | +| ------------------ | -------- | ------------------------------- | +| `granularity_sqla` | _string_ | The SQLA **Time Column** widget | +| `time_grain_sqla` | _string_ | The SQLA **Time Grain** widget | +| `time_range` | _string_ | The **Time range** widget | ### GROUP BY diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index 794f7c910fc90..005cc600ae1bf 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -23,8 +23,8 @@ import logging import os -from cachelib.file import FileSystemCache from celery.schedules import crontab +from flask_caching.backends.filesystemcache import FileSystemCache logger = logging.getLogger() diff --git a/docs/docs/installation/async-queries-celery.mdx b/docs/docs/installation/async-queries-celery.mdx index f6c0dbcc3d7bb..49f1ce72a8b9d 100644 --- a/docs/docs/installation/async-queries-celery.mdx +++ b/docs/docs/installation/async-queries-celery.mdx @@ -66,7 +66,7 @@ celery --app=superset.tasks.celery_app:app beat ``` To setup a result backend, you need to pass an instance of a derivative of from -cachelib.base.BaseCache to the RESULTS_BACKEND configuration key in your superset_config.py. You can +from flask_caching.backends.base import BaseCache to the RESULTS_BACKEND configuration key in your superset_config.py. You can use Memcached, Redis, S3 (https://pypi.python.org/pypi/s3werkzeugcache), memory or the file system (in a single server-type setup or for testing), or to write your own caching interface. Your `superset_config.py` may look something like: @@ -79,7 +79,7 @@ S3_CACHE_KEY_PREFIX = 'sql_lab_result' RESULTS_BACKEND = S3Cache(S3_CACHE_BUCKET, S3_CACHE_KEY_PREFIX) # On Redis -from cachelib.redis import RedisCache +from flask_caching.backends.rediscache import RedisCache RESULTS_BACKEND = RedisCache( host='localhost', port=6379, key_prefix='superset_results') ``` diff --git a/helm/superset/templates/_helpers.tpl b/helm/superset/templates/_helpers.tpl index f7b23634e18e3..b450ec3ef07b5 100644 --- a/helm/superset/templates/_helpers.tpl +++ b/helm/superset/templates/_helpers.tpl @@ -63,7 +63,7 @@ Create chart name and version as used by the chart label. {{- define "superset-config" }} import os -from cachelib.redis import RedisCache +from flask_caching.backends.rediscache import RedisCache def env(key, default=None): return os.getenv(key, default) diff --git a/requirements/base.txt b/requirements/base.txt index d54bd6e8a976a..4928876a13d75 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -32,9 +32,7 @@ bottleneck==1.3.7 brotli==1.0.9 # via flask-compress cachelib==0.6.0 - # via - # apache-superset - # flask-caching + # via flask-caching celery==5.2.2 # via apache-superset certifi==2023.7.22 diff --git a/setup.cfg b/setup.cfg index 4340cf50c47d0..0d7c0b655fdf6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ combine_as_imports = true include_trailing_comma = true line_length = 88 known_first_party = superset -known_third_party =alembic,apispec,backoff,cachelib,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,msgpack,nh3,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,sqlalchemy_bigquery,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,shillelagh,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml +known_third_party =alembic,apispec,backoff,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,msgpack,nh3,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,sqlalchemy_bigquery,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,shillelagh,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml multi_line_output = 3 order_by_type = false diff --git a/setup.py b/setup.py index bafeda5911097..26d8ebc2afea2 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,6 @@ def get_git_sha() -> str: }, install_requires=[ "backoff>=1.8.0", - "cachelib>=0.4.1,<0.7", "celery>=5.2.2, <6.0.0", "click>=8.0.3", "click-option-group", diff --git a/superset/config.py b/superset/config.py index 3f27deffead1a..e646ca8c45a38 100644 --- a/superset/config.py +++ b/superset/config.py @@ -37,10 +37,10 @@ from typing import Any, Callable, Literal, TYPE_CHECKING, TypedDict import pkg_resources -from cachelib.base import BaseCache from celery.schedules import crontab from flask import Blueprint from flask_appbuilder.security.manager import AUTH_DB +from flask_caching.backends.base import BaseCache from pandas import Series from pandas._libs.parsers import STR_NA_VALUES # pylint: disable=no-name-in-module from sqlalchemy.orm.query import Query diff --git a/superset/extensions/__init__.py b/superset/extensions/__init__.py index f6367067a4795..42daf8205b797 100644 --- a/superset/extensions/__init__.py +++ b/superset/extensions/__init__.py @@ -19,9 +19,9 @@ from typing import Any, Callable, Optional import celery -from cachelib.base import BaseCache from flask import Flask from flask_appbuilder import AppBuilder, SQLA +from flask_caching.backends.base import BaseCache from flask_migrate import Migrate from flask_talisman import Talisman from flask_wtf.csrf import CSRFProtect diff --git a/tests/integration_tests/core_tests.py b/tests/integration_tests/core_tests.py index 2f346564e9b60..a10b3974b31aa 100644 --- a/tests/integration_tests/core_tests.py +++ b/tests/integration_tests/core_tests.py @@ -411,6 +411,7 @@ def test_cache_logging(self): self.get_json_resp(f"/superset/warm_up_cache?slice_id={slc.id}") ck = db.session.query(CacheKey).order_by(CacheKey.id.desc()).first() assert ck.datasource_uid == f"{slc.table.id}__table" + db.session.delete(ck) app.config["STORE_CACHE_KEYS_IN_METADATA_DB"] = store_cache_keys def test_redirect_invalid(self): From 92826772db9bcb949a8f429766f655dc446e625d Mon Sep 17 00:00:00 2001 From: "Michael S. Molina" Date: Wed, 30 Aug 2023 15:33:30 -0300 Subject: [PATCH 3/3] Bumps Helm charts --- helm/superset/Chart.yaml | 2 +- helm/superset/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/superset/Chart.yaml b/helm/superset/Chart.yaml index 7582c560c4805..8af3ae4acf579 100644 --- a/helm/superset/Chart.yaml +++ b/helm/superset/Chart.yaml @@ -29,7 +29,7 @@ maintainers: - name: craig-rueda email: craig@craigrueda.com url: https://github.com/craig-rueda -version: 0.10.5 +version: 0.10.6 dependencies: - name: postgresql version: 12.1.6 diff --git a/helm/superset/README.md b/helm/superset/README.md index 9ee8b9d12cfed..724eb4310160d 100644 --- a/helm/superset/README.md +++ b/helm/superset/README.md @@ -23,7 +23,7 @@ NOTE: This file is generated by helm-docs: https://github.com/norwoodj/helm-docs # superset -![Version: 0.10.5](https://img.shields.io/badge/Version-0.10.5-informational?style=flat-square) +![Version: 0.10.6](https://img.shields.io/badge/Version-0.10.6-informational?style=flat-square) Apache Superset is a modern, enterprise-ready business intelligence web application