Merge branch 'apache:master' into master

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -15,14 +15,9 @@ body:
     id: bug-description
     attributes:
       label: Bug description
-      description: A clear and concise description of what the bug is.
-    validations:
-      required: true
-  - type: textarea
-    id: repro-steps
-    attributes:
-      label: How to reproduce the bug
+      description: A clear description of what the bug is, including reproduction steps and expected behavior.
       placeholder: |
+        The bug is that...
         1. Go to '...'
         2. Click on '....'
         3. Scroll down to '....'

.github/workflows/bashlib.sh

@@ -162,7 +162,11 @@ cypress-run-all() {
     USE_DASHBOARD_FLAG='--use-dashboard'
   fi
 
-  python ../../scripts/cypress_run.py --parallelism $PARALLELISM --parallelism-id $PARALLEL_ID $USE_DASHBOARD_FLAG
+  # UNCOMMENT the next few commands to monitor memory usage
+  # monitor_memory &  # Start memory monitoring in the background
+  # memoryMonitorPid=$!
+  python ../../scripts/cypress_run.py --parallelism $PARALLELISM --parallelism-id $PARALLEL_ID --group $PARALLEL_ID --retries 5 $USE_DASHBOARD_FLAG
+  # kill $memoryMonitorPid
 
   # After job is done, print out Flask log for debugging
   echo "::group::Flask log for default run"
@@ -178,6 +182,21 @@ eyes-storybook-dependencies() {
   say "::endgroup::"
 }
 
+monitor_memory() {
+  # This is a small utility to monitor memory usage. Useful for debugging memory in GHA.
+  # To use wrap your command as follows
+  #
+  # monitor_memory &  # Start memory monitoring in the background
+  # memoryMonitorPid=$!
+  # YOUR_COMMAND_HERE
+  # kill $memoryMonitorPid
+  while true; do
+    echo "$(date) - Top 5 memory-consuming processes:"
+    ps -eo pid,comm,%mem --sort=-%mem | head -n 6  # First line is the header, next 5 are top processes
+    sleep 2
+  done
+}
+
 cypress-run-applitools() {
   cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
 

.github/workflows/superset-e2e.yml

@@ -48,7 +48,8 @@ jobs:
       PYTHONPATH: ${{ github.workspace }}
       REDIS_PORT: 16379
       GITHUB_TOKEN: ${{ github.token }}
-      USE_DASHBOARD: ${{ github.event.inputs.use_dashboard || (github.ref == 'refs/heads/master' && 'true') || 'false' }}
+      # use the dashboard feature when running manually OR merging to master
+      USE_DASHBOARD: ${{ github.event.inputs.use_dashboard == 'true'|| (github.ref == 'refs/heads/master' && 'true') || 'false' }}
     services:
       postgres:
         image: postgres:15-alpine

CODE_OF_CONDUCT.md

@@ -80,9 +80,9 @@ If you believe someone is violating this code of conduct, you may reply to them
 
 Or one of our volunteers:
 
-* [Mark Thomas](http://home.apache.org/~markt/coc.html)
-* [Joan Touzet](http://home.apache.org/~wohali/)
-* [Sharan Foga](http://home.apache.org/~sharan/coc.html)
+* [Mark Thomas](https://www.linkedin.com/in/mark-thomas-b16751158/)
+* [Joan Touzet](https://www.apache.org/foundation/conduct-team/wohali.html)
+* [Sharan Foga](https://www.linkedin.com/in/sfoga/)
 
 If the violation is in documentation or code, for example inappropriate pronoun usage or word choice within official documentation, we ask that people report these privately to the project in question at <private@project.apache.org>, and, if they have sufficient ability within the project, to resolve or remove the concerning material, being mindful of the perspective of the person originally reporting the issue.
 

docs/README.md

@@ -16,6 +16,7 @@ KIND, either express or implied.  See the License for the
 specific language governing permissions and limitations
 under the License.
 -->
+
 This is the public documentation site for Superset, built using
 [Docusaurus 2](https://docusaurus.io/). See
 [CONTRIBUTING.md](../CONTRIBUTING.md#documentation) for documentation on

docs/docs/faq.mdx

@@ -174,13 +174,16 @@ You can take a look at this Flask-AppBuilder
 ## Is there a way to force the dashboard to use specific colors?
 
 It is possible on a per-dashboard basis by providing a mapping of labels to colors in the JSON
-Metadata attribute using the `label_colors` key.
+Metadata attribute using the `label_colors` key. You can use either the full hex color, a named color,
+like `red`, `coral` or `lightblue`, or the index in the current color palette (0 for first color, 1 for
+second etc). Example:
 
 ```json
 {
     "label_colors": {
-        "Girls": "#FF69B4",
-        "Boys": "#ADD8E6"
+        "foo": "#FF69B4",
+        "bar": "lightblue",
+        "baz": 0
     }
 }
 ```

docs/package.json

@@ -19,7 +19,7 @@
   "dependencies": {
     "@algolia/client-search": "^4.24.0",
     "@ant-design/icons": "^5.4.0",
-    "@docsearch/react": "^3.6.1",
+    "@docsearch/react": "^3.6.2",
     "@docusaurus/core": "^3.5.2",
     "@docusaurus/plugin-client-redirects": "^3.5.2",
     "@docusaurus/preset-classic": "^3.5.2",
@@ -48,8 +48,8 @@
   "devDependencies": {
     "@docusaurus/module-type-aliases": "^3.5.2",
     "@docusaurus/tsconfig": "^3.5.2",
-    "@types/react": "^18.3.5",
-    "typescript": "^5.5.4",
+    "@types/react": "^18.3.10",
+    "typescript": "^5.6.2",
     "webpack": "^5.94.0"
   },
   "browserslist": {

docs/yarn.lock

@@ -1335,19 +1335,19 @@
   resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz#1d572bfbbe14b7704e0ba0f39b74815b84870d70"
   integrity sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==
 
-"@docsearch/css@3.6.1":
-  version "3.6.1"
-  resolved "https://registry.yarnpkg.com/@docsearch/css/-/css-3.6.1.tgz#f0a728ecb486c81f2d282650fc1820c914913408"
-  integrity sha512-VtVb5DS+0hRIprU2CO6ZQjK2Zg4QU5HrDM1+ix6rT0umsYvFvatMAnf97NHZlVWDaaLlx7GRfR/7FikANiM2Fg==
+"@docsearch/css@3.6.2":
+  version "3.6.2"
+  resolved "https://registry.yarnpkg.com/@docsearch/css/-/css-3.6.2.tgz#ccd9c83dbfeaf34efe4e3547ee596714ae7e5891"
+  integrity sha512-vKNZepO2j7MrYBTZIGXvlUOIR+v9KRf70FApRgovWrj3GTs1EITz/Xb0AOlm1xsQBp16clVZj1SY/qaOJbQtZw==
 
-"@docsearch/react@^3.5.2", "@docsearch/react@^3.6.1":
-  version "3.6.1"
-  resolved "https://registry.yarnpkg.com/@docsearch/react/-/react-3.6.1.tgz#0f826df08693293806d64277d6d9c38636211b97"
-  integrity sha512-qXZkEPvybVhSXj0K7U3bXc233tk5e8PfhoZ6MhPOiik/qUQxYC+Dn9DnoS7CxHQQhHfCvTiN0eY9M12oRghEXw==
+"@docsearch/react@^3.5.2", "@docsearch/react@^3.6.2":
+  version "3.6.2"
+  resolved "https://registry.yarnpkg.com/@docsearch/react/-/react-3.6.2.tgz#32b16dd7d5614f0d39e6bc018549816b68d171b8"
+  integrity sha512-rtZce46OOkVflCQH71IdbXSFK+S8iJZlUF56XBW5rIgx/eG5qoomC7Ag3anZson1bBac/JFQn7XOBfved/IMRA==
   dependencies:
     "@algolia/autocomplete-core" "1.9.3"
     "@algolia/autocomplete-preset-algolia" "1.9.3"
-    "@docsearch/css" "3.6.1"
+    "@docsearch/css" "3.6.2"
     algoliasearch "^4.19.1"
 
 "@docusaurus/core@3.5.2", "@docusaurus/core@^3.5.2":
@@ -2939,10 +2939,10 @@
     "@types/history" "^4.7.11"
     "@types/react" "*"
 
-"@types/react@*", "@types/react@^18.3.5":
-  version "18.3.5"
-  resolved "https://registry.yarnpkg.com/@types/react/-/react-18.3.5.tgz#5f524c2ad2089c0ff372bbdabc77ca2c4dbadf8f"
-  integrity sha512-WeqMfGJLGuLCqHGYRGHxnKrXcTitc6L/nBUWfWPcTarG3t9PsquqUMuVeXZeca+mglY4Vo5GZjCi0A3Or2lnxA==
+"@types/react@*", "@types/react@^18.3.10":
+  version "18.3.10"
+  resolved "https://registry.yarnpkg.com/@types/react/-/react-18.3.10.tgz#6edc26dc22ff8c9c226d3c7bf8357b013c842219"
+  integrity sha512-02sAAlBnP39JgXwkAq3PeU9DVaaGpZyF3MGcC0MKgQVkZor5IiiDAipVaxQHtDJAmO4GIy/rVBy/LzVj76Cyqg==
   dependencies:
     "@types/prop-types" "*"
     csstype "^3.0.2"
@@ -9984,10 +9984,10 @@ types-ramda@^0.30.0:
   dependencies:
     ts-toolbelt "^9.6.0"
 
-typescript@^5.5.4:
-  version "5.5.4"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.5.4.tgz#d9852d6c82bad2d2eda4fd74a5762a8f5909e9ba"
-  integrity sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==
+typescript@^5.6.2:
+  version "5.6.2"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.6.2.tgz#d1de67b6bef77c41823f822df8f0b3bcff60a5a0"
+  integrity sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==
 
 undici-types@~5.26.4:
   version "5.26.5"

pyproject.toml

@@ -57,6 +57,7 @@ dependencies = [
     "geopy",
     "gunicorn>=22.0.0; sys_platform != 'win32'",
     "hashids>=1.3.1, <2",
+    # known issue with holidays 0.26.0 and above related to prophet lib #25017
     "holidays>=0.25, <0.26",
     "humanize",
     "importlib_metadata",
@@ -89,7 +90,7 @@ dependencies = [
     "slack_sdk>=3.19.0, <4",
     "sqlalchemy>=1.4, <2",
     "sqlalchemy-utils>=0.38.3, <0.39",
-    "sqlglot>=23.0.2,<24",
+    "sqlglot>=25.24.0,<26",
     "sqlparse>=0.5.0",
     "tabulate>=0.8.9, <0.9",
     "typing-extensions>=4, <5",

requirements/base.txt

@@ -350,7 +350,7 @@ sqlalchemy-utils==0.38.3
     # via
     #   apache-superset
     #   flask-appbuilder
-sqlglot==23.6.3
+sqlglot==25.24.0
     # via apache-superset
 sqlparse==0.5.0
     # via apache-superset

scripts/cypress_run.py

@@ -16,26 +16,18 @@
 # under the License.
 
 import argparse
-import hashlib
 import os
 import subprocess
 from datetime import datetime
 
 XVFB_PRE_CMD = "xvfb-run --auto-servernum --server-args='-screen 0, 1024x768x24' "
 REPO = os.getenv("GITHUB_REPOSITORY") or "apache/superset"
-GITHUB_EVENT_NAME = os.getenv("GITHUB_REPOSITORY") or "push"
+GITHUB_EVENT_NAME = os.getenv("GITHUB_EVENT_NAME") or "push"
 CYPRESS_RECORD_KEY = os.getenv("CYPRESS_RECORD_KEY") or ""
 
 
-def compute_hash(file_path: str) -> str:
-    return hashlib.md5(file_path.encode()).hexdigest()
-
-
-def compute_group_index(hash_value: str, num_groups: int) -> int:
-    return int(hash_value, 16) % num_groups
-
-
 def generate_build_id() -> str:
+    """Generates a build ID based on the current timestamp."""
     now = datetime.now()
     rounded_minute = now.minute - (now.minute % 20)
     rounded_time = now.replace(minute=rounded_minute, second=0, microsecond=0)
@@ -44,47 +36,72 @@ def generate_build_id() -> str:
     )
 
 
-def get_cypress_cmd(
-    spec_list: list[str], _filter: str, group: str, use_dashboard: bool
-) -> str:
+def run_cypress_for_test_file(
+    test_file: str, retries: int, use_dashboard: bool, group: str, dry_run: bool, i: int
+) -> int:
+    """Runs Cypress for a single test file and retries upon failure."""
     cypress_cmd = "./node_modules/.bin/cypress run"
-
     os.environ["TERM"] = "xterm"
     os.environ["ELECTRON_DISABLE_GPU"] = "true"
     build_id = generate_build_id()
     browser = os.getenv("CYPRESS_BROWSER", "chrome")
-
-    # Add --disable-dev-shm-usage for Chrome browser
     chrome_flags = "--disable-dev-shm-usage"
 
-    if use_dashboard:
-        # Run using cypress.io service
-        spec: str = "cypress/e2e/*/**/*"
-        cmd = (
-            f"{XVFB_PRE_CMD} "
-            f'{cypress_cmd} --spec "{spec}" --browser {browser} '
-            f"--record --group {group} --tag {REPO},{GITHUB_EVENT_NAME} "
-            f"--parallel --ci-build-id {build_id} "
-            f"-- {chrome_flags}"
+    for attempt in range(retries):
+        # Create Cypress command for a single test file
+        cmd: str = ""
+        if use_dashboard:
+            # If/when we want to use cypress' dashboard feature to record the run
+            group_id = f"matrix{group}-file{i}-{attempt}"
+            cmd = (
+                f"{XVFB_PRE_CMD} "
+                f'{cypress_cmd} --spec "{test_file}" --browser {browser} '
+                f"--record --group {group_id} --tag {REPO},{GITHUB_EVENT_NAME} "
+                f"--ci-build-id {build_id} "
+                f"-- {chrome_flags}"
+            )
+        else:
+            os.environ.pop("CYPRESS_RECORD_KEY", None)
+            cmd = (
+                f"{XVFB_PRE_CMD} "
+                f"{cypress_cmd} --browser {browser} "
+                f'--spec "{test_file}" '
+                f"-- {chrome_flags}"
+            )
+            print(f"RUN: {cmd} (Attempt {attempt + 1}/{retries})")
+        if dry_run:
+            # Print the command instead of executing it
+            print(f"DRY RUN: {cmd}")
+            return 0
+
+        process = subprocess.Popen(
+            cmd,
+            shell=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
         )
-    else:
-        # Run local, but split the execution
-        os.environ.pop("CYPRESS_RECORD_KEY", None)
-        spec_list_str = ",".join(sorted(spec_list))
-        if _filter:
-            spec_list_str = ",".join(sorted([s for s in spec_list if _filter in s]))
-        cmd = (
-            f"{XVFB_PRE_CMD} "
-            f"{cypress_cmd} --browser {browser} "
-            f'--spec "{spec_list_str}" '
-            f"-- {chrome_flags}"
-        )
-    return cmd
+
+        # Stream stdout in real-time
+        if process.stdout:
+            for stdout_line in iter(process.stdout.readline, ""):
+                print(stdout_line, end="")
+
+        process.wait()
+
+        if process.returncode == 0:
+            print(f"Test {test_file} succeeded on attempt {attempt + 1}")
+            return 0
+        else:
+            print(f"Test {test_file} failed on attempt {attempt + 1}")
+
+    print(f"Test {test_file} failed after {retries} retries.")
+    return process.returncode
 
 
 def main() -> None:
     parser = argparse.ArgumentParser(
-        description="Generate Cypress commands based on test file hash"
+        description="Run Cypress tests with retries per test file"
     )
     parser.add_argument(
         "--use-dashboard",
@@ -98,9 +115,12 @@ def main() -> None:
         "--parallelism-id", type=int, required=True, help="ID of the parallelism group"
     )
     parser.add_argument(
-        "--filter", type=str, required=False, default=None, help="filter to test"
+        "--filter", type=str, required=False, default=None, help="Filter to test"
     )
     parser.add_argument("--group", type=str, default="Default", help="Group name")
+    parser.add_argument(
+        "--retries", type=int, default=3, help="Number of retries per test file"
+    )
     parser.add_argument(
         "--dry-run",
         action="store_true",
@@ -114,14 +134,17 @@ def main() -> None:
     cypress_tests_path = os.path.join(cypress_base_full_path, "cypress/e2e")
 
     test_files = []
+    file_count = 0
     for root, _, files in os.walk(cypress_tests_path):
         for file in files:
             if file.endswith("test.ts") or file.endswith("test.js"):
+                file_count += 1
                 test_files.append(
                     os.path.join(root, file).replace(cypress_base_full_path, "")
                 )
+    print(f"Found {file_count} test files.")
 
-    # Initialize groups
+    # Initialize groups for round-robin distribution
     groups: dict[int, list[str]] = {i: [] for i in range(args.parallelism)}
 
     # Sort test files to ensure deterministic distribution
@@ -132,12 +155,21 @@ def main() -> None:
         group_index = index % args.parallelism
         groups[group_index].append(test_file)
 
+    # Only run tests for the group that matches the parallelism ID
     group_id = args.parallelism_id
     spec_list = groups[group_id]
-    cmd = get_cypress_cmd(spec_list, args.filter, args.group, args.use_dashboard)
-    print(f"RUN: {cmd}")
-    if not args.dry_run:
-        subprocess.run(cmd, shell=True, check=True, stdout=None, stderr=None)
+
+    # Run each test file independently with retry logic or dry-run
+    processed_file_count: int = 0
+    for i, test_file in enumerate(spec_list):
+        result = run_cypress_for_test_file(
+            test_file, args.retries, args.use_dashboard, args.group, args.dry_run, i
+        )
+        if result != 0:
+            print(f"Exiting due to failure in {test_file}")
+            exit(result)
+        processed_file_count += 1
+    print(f"Ran {processed_file_count} test files successfully.")
 
 
 if __name__ == "__main__":