feat(native-filters): Add legacy (filter-box) to native filter migrat…

…ion script (#23269)

docs/docs/miscellaneous/native-filter-migration.mdx

@@ -0,0 +1,103 @@
+---
+title: Migrating from Legacy to Native Filters
+sidebar_position: 5
+version: 1
+---
+
+##
+
+The `superset native-filters` CLI command group—somewhat akin to an Alembic migration—
+comprises of a number of sub-commands which allows administrators to upgrade/downgrade
+existing dashboards which use the legacy filter-box charts—in combination with the
+filter scopes/filter mapping—to use the native filter dashboard component.
+
+Even though both legacy and native filters can coexist the overall user experience (UX)
+is substandard as the already convoluted filter space becomes overly complex. After
+enabling the `DASHBOARD_NATIVE_FILTERS` it is strongly advised to run the migration ASAP to
+ensure users are not exposed to the hybrid state.
+
+### Upgrading
+
+The
+
+```
+superset native-filters upgrade
+```
+
+command—which provides the option to target either specific dashboard(s) or all
+dashboards—migrates the legacy filters to native filters.
+
+Specifically, the command performs the following:
+
+- Replaces every filter-box chart within the dashboard with a markdown element which
+provides a link to the deprecated chart. This preserves the layout whilst simultaneously
+providing context to help owners review/verify said change.
+- Migrates the filter scopes/filter mappings to the native filter configuration.
+
+#### Quality Control
+
+Dashboard owners should:
+
+- Verify that the filter behavior is correct.
+- Consolidate any conflicting/redundant filters—this previously may not have been
+obvious given the embedded nature of the legacy filters and/or the non-optimal UX of the
+legacy filter mapping (scopes and immunity).
+- Rename the filters—which may not be uniquely named—to provide the necessary context
+which previously was likely provided by both the location of the filter-box and the
+corresponding filter-box title.
+
+Dashboard owners may:
+
+- Remove† the markdown elements from their dashboards and adjust the layout accordingly.
+
+† Note removing the markdown elements—which contain metadata relating to the replaced
+chart—prevents the dashboard from being fully restored and thus this operation should
+only be performed if it is evident that a downgrade is not necessary.
+
+### Downgrading
+
+Similarly the
+
+```
+superset native-filters downgrade
+```
+
+command reverses said migration, i.e., restores the dashboard to the previous state.
+
+
+### Cleanup
+
+The ability to downgrade/reverse the migration requires temporary storage of the
+dashboard metadata—relating to both positional composition and filter configuration.
+
+Once the upgrade has been verified it is recommended to run the
+
+```
+superset native-filters cleanup
+```
+
+command—which provides the option to target either specific dashboard(s) or all
+dashboards. Note this operation is irreversible.
+
+Specifically, the command performs the following:
+
+- Removes the temporary dashboard metadata.
+- Deletes the filter-box charts associated with the dashboard†.
+
+† Note the markdown elements will still remain however the link to the referenced filter-box
+chart will no longer be valid.
+
+Finally, the
+
+```
+superset native-filers cleanup --all
+```
+
+command will additionally delete all filter-box charts, irrespective of whether they
+were ever associated with a dashboard.
+
+#### Quality Control
+
+Dashboard owners should:
+
+- Remove the markdown elements from their dashboards and adjust the layout accordingly.

requirements/base.txt

@@ -40,12 +40,15 @@ click==8.0.4
     #   apache-superset
     #   celery
     #   click-didyoumean
+    #   click-option-group
     #   click-plugins
     #   click-repl
     #   flask
     #   flask-appbuilder
 click-didyoumean==0.3.0
     # via celery
+click-option-group==0.5.5
+    # via apache-superset
 click-plugins==1.1.1
     # via celery
 click-repl==0.2.0
@@ -64,6 +67,8 @@ cryptography==39.0.1
     # via
     #   apache-superset
     #   paramiko
+deprecated==1.2.13
+    # via limits
 deprecation==2.1.0
     # via apache-superset
 dnspython==2.1.0
@@ -78,6 +83,7 @@ flask==2.1.3
     #   flask-caching
     #   flask-compress
     #   flask-jwt-extended
+    #   flask-limiter
     #   flask-login
     #   flask-migrate
     #   flask-sqlalchemy
@@ -92,6 +98,8 @@ flask-compress==1.13
     # via apache-superset
 flask-jwt-extended==4.3.1
     # via flask-appbuilder
+flask-limiter==3.3.0
+    # via flask-appbuilder
 flask-login==0.6.0
     # via
     #   apache-superset
@@ -128,6 +136,8 @@ humanize==3.11.0
     # via apache-superset
 idna==3.2
     # via email-validator
+importlib-metadata==6.0.0
+    # via flask
 isodate==0.6.0
     # via apache-superset
 itsdangerous==2.1.1
@@ -144,10 +154,14 @@ kombu==5.2.4
     # via celery
 korean-lunar-calendar==0.2.1
     # via holidays
+limits==3.2.0
+    # via flask-limiter
 mako==1.1.4
     # via alembic
 markdown==3.3.4
     # via apache-superset
+markdown-it-py==2.2.0
+    # via rich
 markupsafe==2.1.1
     # via
     #   jinja2
@@ -162,17 +176,22 @@ marshmallow-enum==1.5.1
     # via flask-appbuilder
 marshmallow-sqlalchemy==0.23.1
     # via flask-appbuilder
+mdurl==0.1.2
+    # via markdown-it-py
 msgpack==1.0.2
     # via apache-superset
 numpy==1.23.5
     # via
     #   apache-superset
     #   pandas
     #   pyarrow
+ordered-set==4.1.0
+    # via flask-limiter
 packaging==21.3
     # via
     #   bleach
     #   deprecation
+    #   limits
 pandas==1.5.3
     # via apache-superset
 paramiko==2.11.0
@@ -191,6 +210,8 @@ pyarrow==10.0.1
     # via apache-superset
 pycparser==2.20
     # via cffi
+pygments==2.14.0
+    # via rich
 pyjwt==2.4.0
     # via
     #   apache-superset
@@ -232,8 +253,12 @@ pyyaml==5.4.1
     #   apispec
 redis==3.5.3
     # via apache-superset
+rich==13.3.1
+    # via flask-limiter
 selenium==3.141.0
     # via apache-superset
+shortid==0.1.2
+    # via apache-superset
 simplejson==3.17.3
     # via apache-superset
 six==1.16.0
@@ -269,7 +294,11 @@ sshtunnel==0.4.0
 tabulate==0.8.9
     # via apache-superset
 typing-extensions==4.4.0
-    # via apache-superset
+    # via
+    #   apache-superset
+    #   flask-limiter
+    #   limits
+    #   rich
 urllib3==1.26.6
     # via selenium
 vine==5.0.0
@@ -286,6 +315,8 @@ werkzeug==2.1.2
     #   flask
     #   flask-jwt-extended
     #   flask-login
+wrapt==1.12.1
+    # via deprecated
 wtforms==2.3.3
     # via
     #   apache-superset
@@ -296,6 +327,8 @@ wtforms-json==0.3.3
     # via apache-superset
 xlsxwriter==3.0.7
     # via apache-superset
+zipp==3.15.0
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements/development.txt

@@ -80,8 +80,6 @@ pure-sasl==0.6.2
     # via thrift-sasl
 pydruid==0.6.5
     # via apache-superset
-pygments==2.12.0
-    # via ipython
 pyhive[hive]==0.6.5
     # via apache-superset
 pyinstrument==4.0.2

requirements/integration.txt

@@ -30,15 +30,15 @@ packaging==21.3
 pep517==0.11.0
     # via build
 pip-compile-multi==2.6.2
-    # via -r integration.in
+    # via -r requirements/integration.in
 pip-tools==6.8.0
     # via pip-compile-multi
 platformdirs==2.6.2
     # via virtualenv
 pluggy==0.13.1
     # via tox
 pre-commit==3.2.2
-    # via -r integration.in
+    # via -r requirements/integration.in
 py==1.10.0
     # via tox
 pyparsing==3.0.6
@@ -50,11 +50,11 @@ six==1.16.0
 toml==0.10.2
     # via tox
 tomli==1.2.1
-    # via pep517
+    # via build
 toposort==1.6
     # via pip-compile-multi
 tox==3.25.1
-    # via -r integration.in
+    # via -r requirements/integration.in
 virtualenv==20.17.1
     # via
     #   pre-commit

setup.py

@@ -77,6 +77,7 @@ def get_git_sha() -> str:
         "cachelib>=0.4.1,<0.5",
         "celery>=5.2.2, <6.0.0",
         "click>=8.0.3",
+        "click-option-group",
         "colorama",
         "croniter>=0.3.28",
         "cron-descriptor",
@@ -114,6 +115,7 @@ def get_git_sha() -> str:
         "PyJWT>=2.4.0, <3.0",
         "redis",
         "selenium>=3.141.0",
+        "shortid",
         "sshtunnel>=0.4.0, <0.5",
         "simplejson>=3.15.0",
         "slack_sdk>=3.1.1, <4",

superset/cli/main.py

@@ -50,9 +50,12 @@ def make_shell_context() -> Dict[str, Any]:
 ):
     module = importlib.import_module(module_name)
     for attribute in module.__dict__.values():
-        if isinstance(attribute, click.core.Command):
+        if isinstance(attribute, (click.core.Command, click.core.Group)):
             superset.add_command(attribute)
 
+            if isinstance(attribute, click.core.Group):
+                break
+
 
 @superset.command()
 @with_appcontext