Merge branch 'master' into fix/47160-database-validation

.github/CODEOWNERS

@@ -17,3 +17,6 @@
 # Notify Helm Chart maintainers about changes in it
 
 /helm/superset/ @craig-rueda @dpgaspar @villebro
+
+# Notify E2E test maintainers of changes
+/superset-frontend/cypress-base/ @jinghua-qa

.github/dependabot.yml

@@ -7,6 +7,7 @@ updates:
     labels:
       - npm
       - dependabot
+    versioning-strategy: increase
 
   - package-ecosystem: "pip"
     directory: "/requirements/"
@@ -21,9 +22,29 @@ updates:
     schedule:
       interval: "daily"
     open-pull-requests-limit: 0
+    versioning-strategy: increase
 
   - package-ecosystem: "npm"
     directory: "/docs/"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 0
+    versioning-strategy: increase
+
+  - package-ecosystem: "npm"
+    directory: "/superset-websocket/"
+    schedule:
+      interval: "daily"
+    labels:
+      - npm
+      - dependabot
+    versioning-strategy: increase
+
+  - package-ecosystem: "npm"
+    directory: "/superset-websocket/utils/client-ws-app/"
+    schedule:
+      interval: "daily"
+    labels:
+      - npm
+      - dependabot
+    versioning-strategy: increase

.github/workflows/bashlib.sh

@@ -183,7 +183,7 @@ cypress-run-all() {
   nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
   local flaskProcessId=$!
 
-  cypress-run "*/**/!(*.applitools.test.ts)"
+  cypress-run "*/**/*"
 
   # After job is done, print out Flask log for debugging
   say "::group::Flask log for default run"
@@ -198,7 +198,7 @@ cypress-run-all() {
   nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
   local flaskProcessId=$!
 
-  cypress-run "sqllab/!(*.applitools.test.ts)" "Backend persist"
+  cypress-run "sqllab/*" "Backend persist"
 
   # Upload code coverage separately so each page can have separate flags
   # -c will clean existing coverage reports, -F means add flags
@@ -220,14 +220,19 @@ eyes-storybook-dependencies() {
 }
 
 cypress-run-applitools() {
+  cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
+
   local flasklog="${HOME}/flask.log"
   local port=8081
+  local cypress="./node_modules/.bin/cypress run"
+  local browser=${CYPRESS_BROWSER:-chrome}
+
   export CYPRESS_BASE_URL="http://localhost:${port}"
 
   nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
   local flaskProcessId=$!
 
-  cypress-run "*/**/*.applitools.test.ts"
+  $cypress --spec "cypress/integration/*/**/*.applitools.test.ts" --browser "$browser" --headless --config ignoreTestFiles="[]"
 
   codecov -c -F "cypress" || true
 

.github/workflows/superset-frontend.yml

@@ -51,6 +51,11 @@ jobs:
         if: steps.check.outcome == 'failure'
         working-directory: ./superset-frontend
         run: npm run plugins:build-storybook
+      - name: superset-ui/core coverage
+        if: steps.check.outcome == 'failure'
+        working-directory: ./superset-frontend
+        run: |
+          npm run core:cover
       - name: unit tests
         if: steps.check.outcome == 'failure'
         working-directory: ./superset-frontend

.pylintrc

@@ -134,7 +134,9 @@ include-naming-hint=no
 
 # List of decorators that produce properties, such as abc.abstractproperty. Add
 # to this list to register other decorators that produce valid properties.
-property-classes=abc.abstractproperty
+property-classes=
+    abc.abstractproperty,
+    sqlalchemy.ext.hybrid.hybrid_property
 
 # Regular expression matching correct argument names
 argument-rgx=[a-z_][a-z0-9_]{2,30}$

Makefile

@@ -47,7 +47,7 @@ superset:
 	superset load-examples
 
 	# Install node packages
-	cd superset-frontend; npm install
+	cd superset-frontend; npm ci
 
 update: update-py update-js
 
@@ -101,7 +101,7 @@ node-app:
 
 build-cypress:
 	cd superset-frontend; npm run build-instrumented
-	cd superset-frontend/cypress-base; npm install
+	cd superset-frontend/cypress-base; npm ci
 
 open-cypress:
 	if ! [ $(port) ]; then cd superset-frontend/cypress-base; CYPRESS_BASE_URL=http://localhost:9000 npm run cypress open; fi

README.md

@@ -91,6 +91,7 @@ Here are some of the major database solutions that are supported:
   <img src="superset-frontend/src/assets/images/snowflake.png" alt="snowflake" border="0" width="152" height="46"/>
   <img src="superset-frontend/src/assets/images/trino.png" alt="trino" border="0" width="46" height="46"/>
   <img src="superset-frontend/src/assets/images/presto.png" alt="presto" border="0" width="152" height="46"/>
+  <img src="superset-frontend/src/assets/images/databricks.png" alt="databricks" border="0" width="142" height="22" />
   <img src="superset-frontend/src/assets/images/druid.png" alt="druid" border="0" width="135" height="37" />
   <img src="superset-frontend/src/assets/images/firebolt.png" alt="firebolt" border="0" width="133" height="21.5" />
   <img src="superset-frontend/src/assets/images/timescale.png" alt="timescale" border="0" width="102" height="26.8" />  
@@ -142,24 +143,35 @@ how to set up a development environment.
 
 ## Resources
 
+Superset 2.0!
+- [Superset 2.0 Meetup](https://preset.io/events/superset-2-0-meetup/)
+- [Superset 2.0 Release Notes](https://github.com/apache/superset/tree/master/RELEASING/release-notes-2-0)
+
+Understanding the Superset Points of View
+- [The Case for Dataset-Centric Visualization](https://preset.io/blog/dataset-centric-visualization/)
+- [Understanding the Superset Semantic Layer](https://preset.io/blog/understanding-superset-semantic-layer/)
+
+
 - Getting Started with Superset
   - [Superset in 2 Minutes using Docker Compose](https://superset.apache.org/docs/installation/installing-superset-using-docker-compose#installing-superset-locally-using-docker-compose)
   - [Installing Database Drivers](https://superset.apache.org/docs/databases/docker-add-drivers/)
   - [Building New Database Connectors](https://preset.io/blog/building-database-connector/)
   - [Create Your First Dashboard](https://superset.apache.org/docs/creating-charts-dashboards/first-dashboard)
   - [Comprehensive Tutorial for Contributing Code to Apache Superset
 ](https://preset.io/blog/tutorial-contributing-code-to-apache-superset/)
-- [Documentation for Superset End-Users (by Preset)](https://docs.preset.io/docs/terminology)
+- [Resources to master Superset by Preset](https://preset.io/resources/)
+
 - Deploying Superset
   - [Official Docker image](https://hub.docker.com/r/apache/superset)
   - [Helm Chart](https://github.com/apache/superset/tree/master/helm/superset)
+
 - Recordings of Past [Superset Community Events](https://preset.io/events)
-  - [Live Demo: Interactive Time-series Analysis with Druid and Superset](https://preset.io/events/2021-03-02-interactive-time-series-analysis-with-druid-and-superset/)
+  - [Mixed Time Series Charts](https://preset.io/events/mixed-time-series-visualization-in-superset-workshop/)  
+  - [How the Bing Team Customized Superset for the Internal Self-Serve Data & Analytics Platform](https://preset.io/events/how-the-bing-team-heavily-customized-superset-for-their-internal-data/)
   - [Live Demo: Visualizing MongoDB and Pinot Data using Trino](https://preset.io/events/2021-04-13-visualizing-mongodb-and-pinot-data-using-trino/)
-	- [Superset Contributor Bootcamp](https://preset.io/events/superset-contributor-bootcamp-dec-21/)
 	- [Introduction to the Superset API](https://preset.io/events/introduction-to-the-superset-api/)
-	- [Apache Superset 1.3 Meetup](https://preset.io/events/apache-superset-1-3/)
 	- [Building a Database Connector for Superset](https://preset.io/events/2021-02-16-building-a-database-connector-for-superset/)
+
 - Visualizations
   - [Building Custom Viz Plugins](https://superset.apache.org/docs/installation/building-custom-viz-plugins)
   - [Managing and Deploying Custom Viz Plugins](https://medium.com/nmc-techblog/apache-superset-manage-custom-viz-plugins-in-production-9fde1a708e55)

RELEASING/Dockerfile.from_local_tarball

@@ -30,7 +30,7 @@ RUN apt-get install -y apt-transport-https apt-utils
 # Install superset dependencies
 # https://superset.apache.org/docs/installation/installing-superset-from-scratch
 RUN apt-get install -y build-essential libssl-dev \
-    libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev
+    libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev chromium
 
 # Install nodejs for custom build
 # https://nodejs.org/en/download/package-manager/

RELEASING/Dockerfile.from_svn_tarball

@@ -30,7 +30,7 @@ RUN apt-get install -y apt-transport-https apt-utils
 # Install superset dependencies
 # https://superset.apache.org/docs/installation/installing-superset-from-scratch
 RUN apt-get install -y build-essential libssl-dev \
-    libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev
+    libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev chromium
 
 # Install nodejs for custom build
 # https://nodejs.org/en/download/package-manager/

RELEASING/README.md

@@ -30,6 +30,7 @@ partaking in the process should join the channel.
 
 ## Release notes for recent releases
 
+- [2.0](release-notes-2-0/README.md)
 - [1.5](release-notes-1-5/README.md)
 - [1.4](release-notes-1-4/README.md)
 - [1.3](release-notes-1-3/README.md)
@@ -458,11 +459,14 @@ while requesting access to push packages.
 
 ```bash
 twine upload dist/apache-superset-${SUPERSET_VERSION}.tar.gz
-
-# Set your username to token
-# Set your password to the token value, including the pypi- prefix
 ```
 
+Set your username to `__token__`
+
+Set your password to the token value, including the `pypi-` prefix
+
+More information on https://pypi.org/help/#apitoken
+
 ### Announcing
 
 Once it's all done, an [ANNOUNCE] thread announcing the release to the dev@ mailing list is the final step.

RELEASING/release-notes-2-0/README.md

@@ -0,0 +1,152 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Release Notes for Superset 2.0
+
+Superset 2.0 is a big step forward. This release cleans up many legacy code paths and feature flags, and deprecates lots of legacy behaviors in Superset.
+
+- [**Developer Experience**](#developer-experience)
+- [**Features**](#features)
+- [**Config and Feature flags**](#config-and-feature-flags)
+- [**Breaking Changes**](#breaking-changes)
+
+## Developer Experience
+
+- Addition of a statsd guage metric for Slack and email notifications for increased visibility into errors around alerts / reports ([#20158](https://github.com/apache/superset/pull/20158))
+
+- Helm chart now supports resource limits and requests for each component ([#20052](https://github.com/apache/superset/pull/20052))
+
+- New Github workflow to test Storybook Netlify instance nightly ([#19852](https://github.com/apache/superset/pull/19852))
+
+- Minimum requirement for Superset is now Python 3.8 ([#19017](https://github.com/apache/superset/pull/19017)
+
+## Features
+
+**Charting and Dashboard Experience**
+
+Support for horizontal bar chart added ([#19918](https://github.com/apache/superset/pull/19918))
+
+![horizontal](https://user-images.githubusercontent.com/11830681/166248149-4946388a-5051-4d13-a516-50a81e9b5be3.png)
+
+Time Series Charts now support stacking of both negative and positive values ([#20408](https://github.com/apache/superset/pull/20408))
+
+![negative](https://user-images.githubusercontent.com/15073128/174057996-52255bfe-60c3-4727-be99-e328c124e439.png)
+
+- Pie charts now defaults to a row limit of 100 to prevent crashes when a high-cardinality column is chosen as the dimension ([#20392](https://github.com/apache/superset/pull/20392))
+
+- World map chart now supports coloring either by metric or by the country column ([#19881](https://github.com/apache/superset/pull/19881))
+
+- Table visualization now supports drag and drop for columns ([#19381](https://github.com/apache/superset/pull/19381))
+
+- Mixed chart now supports Advanced Analytics ([#19851](https://github.com/apache/superset/pull/19851))
+
+- Add support for generic x-axis (non-time-series) in the Mixed Chart ([#20097](https://github.com/apache/superset/pull/20097))
+
+![Image](https://user-images.githubusercontent.com/33317356/168807749-b021c04c-8902-4b4f-a7a4-f21544fb678e.png)
+
+- Charts can now be created in Edit Dashboard mode ([#20126](https://github.com/apache/superset/pull/20126))
+
+![Image](https://user-images.githubusercontent.com/15073128/169251205-2c0f36bb-17e0-4549-aa84-66a58a377603.png)
+
+- Add aggregate total for Pie charts ([#19622](https://github.com/apache/superset/pull/19622))
+
+- Legend is now enabled by default for relevant charts ([#19927](https://github.com/apache/superset/pull/19927))
+
+**View Results Experience**
+
+- Explore and Dashboard views now support displaying of multiple results ([#20277](https://github.com/apache/superset/pull/20277))
+
+- Results pane in Dashboard view now more closely mirrors rich functionality from Results pane in Explore ([#20144](https://github.com/apache/superset/pull/20144))
+
+
+**Quality of Life**
+
+- Edit Dataset modal now doesn't close when you click away ([#20278](https://github.com/apache/superset/pull/20278))
+
+- When editing the label in the Metrics popover, pressing Enter now saves what you typed ([#19898](https://github.com/apache/superset/pull/19898))
+
+- When adding new chart from the dashboard view, the dashboard name will now pre-fill ([#20129](https://github.com/apache/superset/pull/20129))
+
+- Annotations now have an improved empty state ([#20160](https://github.com/apache/superset/pull/20160))
+
+- Confirmational modal is now shown if user exits Explore without saving changes ([#19993](https://github.com/apache/superset/pull/19993))
+
+- Explore popovers now close when the Escape key is pressed ([#19902](https://github.com/apache/superset/pull/19902))
+
+- Run and Save buttons are redesigned for improved usability ([#19558](https://github.com/apache/superset/pull/19558))
+
+**Databases**
+
+- Native database driver for Databricks ([#20320](https://github.com/apache/superset/pull/20320))
+
+- Time grains for SQLite are now simplified ([#19745](https://github.com/apache/superset/pull/19745))
+
+- Multiple upgrades to the Trino database engine ([#20152](https://github.com/apache/superset/pull/20152))
+
+- Switch from sqlalchemy-trino to trino-python-client ([#19957](https://github.com/apache/superset/pull/19957))
+
+- Apache Pinot now supports more time grains in Superset ([#19724](https://github.com/apache/superset/pull/19724))
+
+**Jinja**
+
+- New Jinja macro enables querying / referencing both physical and virtual datasets in SQL Lab ([#15241](https://github.com/apache/superset/pull/15241))
+
+- New Jinja macro added to improve experience of including multiple items ([#19574](https://github.com/apache/superset/pull/19574))
+
+**Other**
+
+- Datasets can now be filtered by their certification status ([#20136](https://github.com/apache/superset/pull/20136))
+
+
+## Config and Feature Flags
+
+- Initial implementation of advanced types ([#18794](https://github.com/apache/superset/pull/18794))
+	- Flag: `ENABLE_ADVANCED_DATA_TYPES`
+
+- Caching can now be enabled in database setups when user impersonation is enabled ([#20114](https://github.com/apache/superset/pull/20114))
+	- Flag: `CACHE_IMPERSONATION`
+
+- Control behavior for how color palettes are chosen ([#19987](https://github.com/apache/superset/pull/19987))
+	- Flag: `USE_ANALAGOUS_COLORS`
+
+- Enabling non-time-series x-axis in some charts ([#20097](https://github.com/apache/superset/pull/20097))
+	- Flag: `GENERIC_CHART_AXES`
+
+- As part of the 2.0 cleanup efforts, the following feature flags were removed (which means the behavior is now permanently enabled):
+	- `ROW_LEVEL_SECURITY`
+	- `ENABLE_REACT_CRUD_VIEWS`
+	- `PUBLIC_ROLE_LIKE_GAMMA`
+
+- The following feature flags are now True by default, but can be turned back to False:
+	- `ENABLE_EXPLORE_DRAG_AND_DROP`
+	- `ENABLE_DND_WITH_CLICK_UX`
+	- `DISABLE_LEGACY_DATASOURCE_EDITOR`
+	- `SQLLAB_BACKEND_PERSISTENCE`
+	- `VERSIONED_EXPORT`
+
+- The following config flags were removed:
+	- `APP_ICON_WIDTH`
+
+- A number of legacy interim flags were removed around SIP-15 ([#18936](https://github.com/apache/superset/pull/18936))
+
+- The `ENABLE_JAVASCRIPT_CONTROLS` flag was moved from an app config to a feature flag
+
+## Breaking Changes
+
+To learn more about the breaking changes in 2.0, please read [UPDATING.MD](https://github.com/apache/superset/blob/master/UPDATING.md)