chore(deps): bump pandas >=2.0

requirements/base.txt

@@ -192,7 +192,7 @@ packaging==23.1
     #   deprecation
     #   limits
     #   marshmallow
-pandas==1.5.3
+pandas==2.0.3
     # via apache-superset
 paramiko==2.11.0
     # via sshtunnel
@@ -290,6 +290,8 @@ typing-extensions==4.4.0
     #   apache-superset
     #   flask-limiter
     #   limits
+tzdata==2023.3
+    # via pandas
 urllib3==1.26.6
     # via selenium
 vine==5.0.0

requirements/testing.txt

@@ -148,8 +148,6 @@ tqdm==4.65.0
     #   prophet
 trino==0.324.0
     # via apache-superset
-tzdata==2023.3
-    # via pytz-deprecation-shim
 tzlocal==4.3
     # via trino
 websocket-client==1.5.1

setup.py

@@ -102,7 +102,7 @@ def get_git_sha() -> str:
         "nh3>=0.2.11, <0.3",
         "numpy==1.23.5",
         "packaging",
-        "pandas>=1.5.3, <1.6",
+        "pandas>=2.0.3, <2.1",
         "parsedatetime",
         "pgsanity",
         "polyline>=2.0.0, <3.0",

superset/common/query_context_processor.py

@@ -134,17 +134,15 @@ def get_df_payload(
 
         if query_obj and cache_key and not cache.is_loaded:
             try:
-                invalid_columns = [
+                if invalid_columns := [
                     col
                     for col in get_column_names_from_columns(query_obj.columns)
                     + get_column_names_from_metrics(query_obj.metrics or [])
                     if (
                         col not in self._qc_datasource.column_names
                         and col != DTTM_ALIAS
                     )
-                ]
-
-                if invalid_columns:
+                ]:
                     raise QueryObjectValidationError(
                         _(
                             "Columns missing in dataset: %(invalid_columns)s",
@@ -570,7 +568,7 @@ def get_data(self, df: pd.DataFrame) -> str | list[dict[str, Any]]:
                     df, index=include_index, **config["CSV_EXPORT"]
                 )
             elif self._query_context.result_format == ChartDataResultFormat.XLSX:
-                result = excel.df_to_excel(df, **config["EXCEL_EXPORT"])
+                result = excel.df_to_excel(df)
             return result or ""
 
         return df.to_dict(orient="records")

superset/config.py

@@ -758,11 +758,6 @@ class D3Format(TypedDict, total=False):
 # note: index option should not be overridden
 CSV_EXPORT = {"encoding": "utf-8"}
 
-# Excel Options: key/value pairs that will be passed as argument to DataFrame.to_excel
-# method.
-# note: index option should not be overridden
-EXCEL_EXPORT = {"encoding": "utf-8"}
-
 # ---------------------------------------------------
 # Time grain configurations
 # ---------------------------------------------------

superset/reports/notifications/slack.py

@@ -21,6 +21,7 @@
 from typing import Union
 
 import backoff
+import pandas as pd
 from flask_babel import gettext as __
 from slack_sdk import WebClient
 from slack_sdk.errors import (
@@ -121,17 +122,19 @@ def _get_body(self) -> str:
         # need to truncate the data
         for i in range(len(df) - 1):
             truncated_df = df[: i + 1].fillna("")
-            truncated_df = truncated_df.append(
-                {k: "..." for k in df.columns}, ignore_index=True
+            truncated_row = pd.Series({k: "..." for k in df.columns})
+            truncated_df = pd.concat(
+                [truncated_df, truncated_row.to_frame().T], ignore_index=True
             )
             tabulated = df.to_markdown()
             table = f"```\n{tabulated}\n```\n\n(table was truncated)"
             message = self._message_template(table)
             if len(message) > MAXIMUM_MESSAGE_SIZE:
                 # Decrement i and build a message that is under the limit
                 truncated_df = df[:i].fillna("")
-                truncated_df = truncated_df.append(
-                    {k: "..." for k in df.columns}, ignore_index=True
+                truncated_row = pd.Series({k: "..." for k in df.columns})
+                truncated_df = pd.concat(
+                    [truncated_df, truncated_row.to_frame().T], ignore_index=True
                 )
                 tabulated = df.to_markdown()
                 table = (

superset/views/database/views.py

@@ -201,7 +201,6 @@ def form_post(self, form: CsvToDatabaseForm) -> Response:
                     infer_datetime_format=form.infer_datetime_format.data,
                     iterator=True,
                     keep_default_na=not form.null_values.data,
-                    mangle_dupe_cols=form.overwrite_duplicate.data,
                     usecols=form.use_cols.data if form.use_cols.data else None,
                     na_values=form.null_values.data if form.null_values.data else None,
                     nrows=form.nrows.data,
@@ -344,7 +343,6 @@ def form_post(self, form: ExcelToDatabaseForm) -> Response:
                 index_col=form.index_col.data,
                 io=form.excel_file.data,
                 keep_default_na=not form.null_values.data,
-                mangle_dupe_cols=form.mangle_dupe_cols.data,
                 na_values=form.null_values.data if form.null_values.data else None,
                 parse_dates=form.parse_dates.data,
                 skiprows=form.skiprows.data,

superset/viz.py

@@ -2849,7 +2849,7 @@ def levels_for(
         for i in range(0, len(groups) + 1):
             agg_df = df.groupby(groups[:i]) if i else df
             levels[i] = (
-                agg_df.mean()
+                agg_df.mean(numeric_only=True)
                 if time_op == "agg_mean"
                 else agg_df.sum(numeric_only=True)
             )
@@ -2874,7 +2874,7 @@ def levels_for_diff(
                 lambda a, b, fill_value: a / float(b) - 1,
             ],
         }[time_op]
-        agg_df = df.groupby(DTTM_ALIAS).sum()
+        agg_df = df.groupby(DTTM_ALIAS).sum(numeric_only=True)
         levels = {
             0: pd.Series(
                 {
@@ -2884,7 +2884,7 @@ def levels_for_diff(
             )
         }
         for i in range(1, len(groups) + 1):
-            agg_df = df.groupby([DTTM_ALIAS] + groups[:i]).sum()
+            agg_df = df.groupby([DTTM_ALIAS] + groups[:i]).sum(numeric_only=True)
             levels[i] = pd.DataFrame(
                 {
                     m: func[0](agg_df[m][until], agg_df[m][since], fill_value=0)
@@ -2900,7 +2900,7 @@ def levels_for_time(
         procs = {}
         for i in range(0, len(groups) + 1):
             self.form_data["groupby"] = groups[:i]
-            df_drop = df.drop(groups[i:], 1)
+            df_drop = df.drop(groups[i:], axis=1)
             procs[i] = self.process_data(df_drop, aggregate=True)
         self.form_data["groupby"] = groups
         return procs

tests/integration_tests/viz_tests.py

@@ -894,7 +894,7 @@ def test_nest_procs_returns_hierarchy(self):
         metrics = ["metric1", "metric2", "metric3"]
         procs = {}
         for i in range(0, 4):
-            df_drop = df.drop(groups[i:], 1)
+            df_drop = df.drop(groups[i:], axis=1)
             pivot = df_drop.pivot_table(
                 index=DTTM_ALIAS, columns=groups[:i], values=metrics
             )

tests/unit_tests/pandas_postprocessing/test_rolling.py

@@ -162,8 +162,8 @@ def test_rolling_after_pivot_with_single_metric():
         pd.DataFrame(
             data={
                 "dttm": pd.to_datetime(["2019-01-01", "2019-01-02"]),
-                FLAT_COLUMN_SEPARATOR.join(["sum_metric", "UK"]): [5.0, 12.0],
-                FLAT_COLUMN_SEPARATOR.join(["sum_metric", "US"]): [6.0, 14.0],
+                FLAT_COLUMN_SEPARATOR.join(["sum_metric", "UK"]): [5, 12],
+                FLAT_COLUMN_SEPARATOR.join(["sum_metric", "US"]): [6, 14],
             }
         )
     )
@@ -213,10 +213,10 @@ def test_rolling_after_pivot_with_multiple_metrics():
         pd.DataFrame(
             data={
                 "dttm": pd.to_datetime(["2019-01-01", "2019-01-02"]),
-                FLAT_COLUMN_SEPARATOR.join(["count_metric", "UK"]): [1.0, 4.0],
-                FLAT_COLUMN_SEPARATOR.join(["count_metric", "US"]): [2.0, 6.0],
-                FLAT_COLUMN_SEPARATOR.join(["sum_metric", "UK"]): [5.0, 12.0],
-                FLAT_COLUMN_SEPARATOR.join(["sum_metric", "US"]): [6.0, 14.0],
+                FLAT_COLUMN_SEPARATOR.join(["count_metric", "UK"]): [1, 4],
+                FLAT_COLUMN_SEPARATOR.join(["count_metric", "US"]): [2, 6],
+                FLAT_COLUMN_SEPARATOR.join(["sum_metric", "UK"]): [5, 12],
+                FLAT_COLUMN_SEPARATOR.join(["sum_metric", "US"]): [6, 14],
             }
         )
     )