diff --git a/UPDATING.md b/UPDATING.md index 8f8c8c7d31c71..9f681475b718b 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -22,6 +22,9 @@ This file documents any backwards-incompatible changes in Superset and assists people when migrating to a new version. ## Next +* [9109](https://github.com/apache/incubator-superset/pull/9109): Expire `filter_immune_slices` and +`filter_immune_filter_fields` to favor dashboard scoped filter metadata `filter_scopes`. + * [9046](https://github.com/apache/incubator-superset/pull/9046): Replaces `can_only_access_owned_queries` by `all_query_access` favoring a white list approach. Since a new permission is introduced use `superset init` to create and associate it by default to the `Admin` role. Note that, by default, all non `Admin` users will diff --git a/superset/migrations/versions/3325d4caccc8_dashboard_scoped_filters.py b/superset/migrations/versions/3325d4caccc8_dashboard_scoped_filters.py new file mode 100644 index 0000000000000..d3a96427cd582 --- /dev/null +++ b/superset/migrations/versions/3325d4caccc8_dashboard_scoped_filters.py @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""empty message + +Revision ID: 3325d4caccc8 +Revises: e96dbf2cfef0 +Create Date: 2020-02-07 14:13:51.714678 + +""" + +# revision identifiers, used by Alembic. +import json +import logging + +from alembic import op +from sqlalchemy import and_, Column, ForeignKey, Integer, String, Table, Text +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship + +from superset import db +from superset.utils.dashboard_filter_scopes_converter import convert_filter_scopes + +revision = "3325d4caccc8" +down_revision = "e96dbf2cfef0" + +Base = declarative_base() + + +class Slice(Base): + """Declarative class to do query in upgrade""" + + __tablename__ = "slices" + id = Column(Integer, primary_key=True) + slice_name = Column(String(250)) + params = Column(Text) + viz_type = Column(String(250)) + + +dashboard_slices = Table( + "dashboard_slices", + Base.metadata, + Column("id", Integer, primary_key=True), + Column("dashboard_id", Integer, ForeignKey("dashboards.id")), + Column("slice_id", Integer, ForeignKey("slices.id")), +) + + +class Dashboard(Base): + __tablename__ = "dashboards" + id = Column(Integer, primary_key=True) + json_metadata = Column(Text) + slices = relationship("Slice", secondary=dashboard_slices, backref="dashboards") + + +def upgrade(): + bind = op.get_bind() + session = db.Session(bind=bind) + + dashboards = session.query(Dashboard).all() + for i, dashboard in enumerate(dashboards): + print(f"scanning dashboard ({i + 1}/{len(dashboards)}) >>>>") + try: + json_metadata = json.loads(dashboard.json_metadata or "{}") + if "filter_scopes" in json_metadata: + continue + + filters = [ + slice for slice in dashboard.slices if slice.viz_type == "filter_box" + ] + + # if dashboard has filter_box + if filters: + filter_scopes = convert_filter_scopes(json_metadata, filters) + json_metadata["filter_scopes"] = filter_scopes + logging.info( + f"Adding filter_scopes for dashboard {dashboard.id}: {json.dumps(filter_scopes)}" + ) + + json_metadata.pop("filter_immune_slices", None) + json_metadata.pop("filter_immune_slice_fields", None) + + if json_metadata: + dashboard.json_metadata = json.dumps( + json_metadata, indent=None, separators=(",", ":"), sort_keys=True + ) + else: + dashboard.json_metadata = None + + session.merge(dashboard) + except Exception as e: + logging.exception(f"dashboard {dashboard.id} has error: {e}") + + session.commit() + session.close() + + +def downgrade(): + pass diff --git a/superset/utils/dashboard_filter_scopes_converter.py b/superset/utils/dashboard_filter_scopes_converter.py new file mode 100644 index 0000000000000..7fff9dd1c5e29 --- /dev/null +++ b/superset/utils/dashboard_filter_scopes_converter.py @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import json +import logging +from collections import defaultdict +from typing import Dict, List + +from superset.models.slice import Slice + +logger = logging.getLogger(__name__) + + +def convert_filter_scopes(json_metadata: Dict, filters: List[Slice]): + filter_scopes = {} + immuned_by_id: List[int] = json_metadata.get("filter_immune_slices") or [] + immuned_by_column: Dict = defaultdict(list) + for slice_id, columns in json_metadata.get( + "filter_immune_slice_fields", {} + ).items(): + for column in columns: + immuned_by_column[column].append(int(slice_id)) + + def add_filter_scope(filter_field, filter_id): + # in case filter field is invalid + if isinstance(filter_field, str): + current_filter_immune = list( + set(immuned_by_id + immuned_by_column.get(filter_field, [])) + ) + filter_fields[filter_field] = { + "scope": ["ROOT_ID"], + "immune": current_filter_immune, + } + else: + logging.info(f"slice [{filter_id}] has invalid field: {filter_field}") + + for filter_slice in filters: + filter_fields: Dict = {} + filter_id = filter_slice.id + slice_params = json.loads(filter_slice.params or "{}") + configs = slice_params.get("filter_configs") or [] + + if slice_params.get("date_filter"): + add_filter_scope("__time_range", filter_id) + if slice_params.get("show_sqla_time_column"): + add_filter_scope("__time_col", filter_id) + if slice_params.get("show_sqla_time_granularity"): + add_filter_scope("__time_grain", filter_id) + if slice_params.get("show_druid_time_granularity"): + add_filter_scope("__granularity", filter_id) + if slice_params.get("show_druid_time_origin"): + add_filter_scope("druid_time_origin", filter_id) + for config in configs: + add_filter_scope(config.get("column"), filter_id) + + if filter_fields: + filter_scopes[filter_id] = filter_fields + + return filter_scopes