feat(python): add openpyxl as a new/optional engine for read_excel

README.md

@@ -206,6 +206,7 @@ You can also install the dependencies directly.
 | fsspec     | Support for reading from remote file systems                                 |
 | connectorx | Support for reading from SQL databases                                       |
 | xlsx2csv   | Support for reading from Excel files                                         |
+| openpyxl   | Support for reading from Excel files with native types                       |
 | deltalake  | Support for reading from Delta Lake Tables                                   |
 | timezone   | Timezone support, only needed if are on Python<3.9 or you are on Windows     |
 

py-polars/polars/io/excel/functions.py

@@ -22,6 +22,7 @@ def read_excel(
     sheet_name: str,
     xlsx2csv_options: dict[str, Any] | None = ...,
     read_csv_options: dict[str, Any] | None = ...,
+    engine: Literal["xlsx2csv", "openpyxl"] | None = ...,
     raise_if_empty: bool = ...,
 ) -> pl.DataFrame:
     ...
@@ -35,6 +36,7 @@ def read_excel(
     sheet_name: None = ...,
     xlsx2csv_options: dict[str, Any] | None = ...,
     read_csv_options: dict[str, Any] | None = ...,
+    engine: Literal["xlsx2csv", "openpyxl"] | None = ...,
     raise_if_empty: bool = ...,
 ) -> pl.DataFrame:
     ...
@@ -48,6 +50,7 @@ def read_excel(
     sheet_name: str,
     xlsx2csv_options: dict[str, Any] | None = ...,
     read_csv_options: dict[str, Any] | None = ...,
+    engine: Literal["xlsx2csv", "openpyxl"] | None = ...,
     raise_if_empty: bool = ...,
 ) -> NoReturn:
     ...
@@ -63,6 +66,7 @@ def read_excel(
     sheet_name: None = ...,
     xlsx2csv_options: dict[str, Any] | None = ...,
     read_csv_options: dict[str, Any] | None = ...,
+    engine: Literal["xlsx2csv", "openpyxl"] | None = ...,
     raise_if_empty: bool = ...,
 ) -> dict[str, pl.DataFrame]:
     ...
@@ -76,6 +80,7 @@ def read_excel(
     sheet_name: None = ...,
     xlsx2csv_options: dict[str, Any] | None = ...,
     read_csv_options: dict[str, Any] | None = ...,
+    engine: Literal["xlsx2csv", "openpyxl"] | None = ...,
     raise_if_empty: bool = ...,
 ) -> pl.DataFrame:
     ...
@@ -88,13 +93,18 @@ def read_excel(
     sheet_name: str | None = None,
     xlsx2csv_options: dict[str, Any] | None = None,
     read_csv_options: dict[str, Any] | None = None,
+    engine: Literal["xlsx2csv", "openpyxl"] | None = None,
     raise_if_empty: bool = True,
 ) -> pl.DataFrame | dict[str, pl.DataFrame]:
     """
     Read Excel (XLSX) sheet into a DataFrame.
 
-    Converts an Excel sheet with ``xlsx2csv.Xlsx2csv().convert()`` to CSV and parses the
-    CSV output with :func:`read_csv`.
+    If using the ``xlsx2csv`` engine, converts an Excel sheet with
+    ``xlsx2csv.Xlsx2csv().convert()`` to CSV and parses the CSV output with
+    :func:`read_csv`.
+
+    When using the ``openpyxl`` engine, reads an Excel sheet with
+    ``openpyxl.load_workbook(source)``.
 
     Parameters
     ----------
@@ -116,6 +126,11 @@ def read_excel(
         ``xlsx2csv.Xlsx2csv().convert()``
         e.g.: ``{"has_header": False, "new_columns": ["a", "b", "c"],
         "infer_schema_length": None}``
+    engine
+        Library used to parse Excel, either openpyxl or xlsx2csv (default is xlsx2csv).
+        Please note that xlsx2csv converts first to csv, making type inference worse
+        than openpyxl. To remedy that, you can use the extra options defined on
+        `xlsx2csv_options` and `read_csv_options`
     raise_if_empty
         When there is no data in the sheet,``NoDataError`` is raised. If this parameter
         is set to False, an empty DataFrame (with no columns) is returned instead.
@@ -155,20 +170,22 @@ def read_excel(
     ...     read_csv_options={"infer_schema_length": None},
     ... )  # doctest: +SKIP
 
+    The ``openpyxl`` engine can also be used to provide automatic type inference.
+    To do so, specify the right engine (`xlsx2csv_options` and `read_csv_options`
+    will be ignored):
+
+    >>> pl.read_excel(
+    ...     source="test.xlsx",
+    ...     engine="openpyxl",
+    ... )  # doctest: +SKIP
+
     If :func:`read_excel` does not work or you need to read other types of
     spreadsheet files, you can try pandas ``pd.read_excel()``
     (supports `xls`, `xlsx`, `xlsm`, `xlsb`, `odf`, `ods` and `odt`).
 
     >>> pl.from_pandas(pd.read_excel("test.xlsx"))  # doctest: +SKIP
 
     """
-    try:
-        import xlsx2csv
-    except ImportError:
-        raise ModuleNotFoundError(
-            "xlsx2csv is not installed\n\nPlease run: `pip install xlsx2csv`"
-        ) from None
-
     if sheet_id is not None and sheet_name is not None:
         raise ValueError(
             f"cannot specify both `sheet_name` ({sheet_name!r}) and `sheet_id` ({sheet_id!r})"
@@ -182,27 +199,50 @@ def read_excel(
     if read_csv_options is None:
         read_csv_options = {}
 
-    # convert sheets to csv
-    parser = xlsx2csv.Xlsx2csv(source, **xlsx2csv_options)
+    reader_fn: Any  # make mypy happy
+    # do conditions imports
+    if engine == "openpyxl":
+        try:
+            import openpyxl
+        except ImportError:
+            raise ImportError(
+                "openpyxl is not installed\n\nPlease run `pip install openpyxl`"
+            ) from None
+        parser = openpyxl.load_workbook(source)
+        sheets = [
+            {"index": i + 1, "name": sheet.title} for i, sheet in enumerate(parser)
+        ]
+        reader_fn = _read_excel_sheet_openpyxl
+    elif engine == "xlsx2csv" or engine is None:  # default
+        try:
+            import xlsx2csv
+        except ImportError:
+            raise ModuleNotFoundError(
+                "xlsx2csv is not installed\n\nPlease run: `pip install xlsx2csv`"
+            ) from None
+        # convert sheets to csv
+        parser = xlsx2csv.Xlsx2csv(source, **xlsx2csv_options)
+        sheets = parser.workbook.sheets
+        reader_fn = _read_excel_sheet_xlsx2csv
+    else:
+        raise NotImplementedError(f"Cannot find the engine `{engine}`")
 
     if sheet_id == 0:
-        # read ALL sheets
         return {
-            sheet["name"]: _read_excel_sheet(
+            sheet["name"]: reader_fn(
                 parser=parser,
                 sheet_id=sheet["index"],
                 sheet_name=None,
                 read_csv_options=read_csv_options,
                 raise_if_empty=raise_if_empty,
             )
-            for sheet in parser.workbook.sheets
+            for sheet in sheets
         }
     else:
         # read a specific sheet by id or name
         if sheet_name is None:
             sheet_id = sheet_id or 1
-
-        return _read_excel_sheet(
+        return reader_fn(
             parser=parser,
             sheet_id=sheet_id,
             sheet_name=sheet_name,
@@ -211,7 +251,38 @@ def read_excel(
         )
 
 
-def _read_excel_sheet(
+def _read_excel_sheet_openpyxl(
+    parser: Any,
+    sheet_id: int | None,
+    sheet_name: str | None,
+    read_csv_options: dict[str, Any] | None,
+    raise_if_empty: bool,
+) -> pl.DataFrame:
+    # read requested sheet if provided on kwargs, otherwise read active sheet
+    if sheet_name is not None:
+        ws = parser[sheet_name]
+    elif sheet_id is not None:
+        ws = parser.worksheets[sheet_id - 1]
+    else:
+        ws = parser.active
+
+    rows_iter = iter(ws.rows)
+
+    # check whether to include or omit the header
+    header = [str(cell.value) for cell in next(rows_iter)]
+
+    df = pl.DataFrame(
+        {key: cell.value for key, cell in zip(header, row)} for row in rows_iter
+    )
+    if raise_if_empty and len(df) == 0:
+        raise NoDataError(
+            "Empty Excel sheet; if you want to read this as "
+            "an empty DataFrame, set `raise_if_empty=False`"
+        )
+    return df
+
+
+def _read_excel_sheet_xlsx2csv(
     parser: Any,
     sheet_id: int | None,
     sheet_name: str | None,

py-polars/pyproject.toml

@@ -44,6 +44,7 @@ numpy = ["numpy >= 1.16.0"]
 fsspec = ["fsspec"]
 connectorx = ["connectorx"]
 xlsx2csv = ["xlsx2csv >= 0.8.0"]
+openpyxl = ["openpyxl >= 3.0.0"]
 deltalake = ["deltalake >= 0.10.0"]
 timezone = ["backports.zoneinfo; python_version < '3.9'", "tzdata; platform_system == 'Windows'"]
 matplotlib = ["matplotlib"]
@@ -84,6 +85,7 @@ module = [
   "sqlalchemy.*",
   "xlsx2csv",
   "xlsxwriter.*",
+  "openpyxl",
   "zoneinfo",
 ]
 ignore_missing_imports = true

py-polars/requirements-dev.txt

@@ -15,6 +15,7 @@ backports.zoneinfo; python_version < '3.9'
 tzdata; platform_system == 'Windows'
 SQLAlchemy
 xlsx2csv
+openpyxl
 XlsxWriter
 adbc_driver_sqlite; python_version >= '3.9' and platform_system != 'Windows'
 connectorx

py-polars/tests/unit/io/test_excel.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from datetime import date
+from datetime import date, datetime
 from io import BytesIO
 from typing import TYPE_CHECKING, Any
 
@@ -42,6 +42,56 @@ def test_read_excel_all_sheets(excel_file_path: Path) -> None:
     assert_frame_equal(df["Sheet2"], expected2)
 
 
+def test_read_excel_all_sheets_openpyxl(excel_file_path: Path) -> None:
+    df = pl.read_excel(excel_file_path, sheet_id=0, engine="openpyxl")
+
+    expected1 = pl.DataFrame({"hello": ["Row 1", "Row 2"]})
+    expected2 = pl.DataFrame({"world": ["Row 3", "Row 4"]})
+
+    assert_frame_equal(df["Sheet1"], expected1)
+    assert_frame_equal(df["Sheet2"], expected2)
+
+
+def test_basic_datatypes_openpyxl_read_excel() -> None:
+    df = pl.DataFrame(
+        {
+            "A": [1, 2, 3, 4, 5],
+            "fruits": ["banana", "banana", "apple", "apple", "banana"],
+            "floats": [1.1, 1.2, 1.3, 1.4, 1.5],
+            "datetime": [datetime(2023, 1, x) for x in range(1, 6)],
+            "nulls": [1, None, None, None, 1],
+        }
+    )
+    xls = BytesIO()
+    df.write_excel(xls)
+    # check if can be read as it was written
+    # we use openpyxl because type inference is better
+    df_by_default = pl.read_excel(xls, engine="openpyxl")
+    df_by_sheet_id = pl.read_excel(xls, sheet_id=1, engine="openpyxl")
+    df_by_sheet_name = pl.read_excel(xls, sheet_name="Sheet1", engine="openpyxl")
+
+    assert_frame_equal(df, df_by_default)
+    assert_frame_equal(df, df_by_sheet_id)
+    assert_frame_equal(df, df_by_sheet_name)
+
+
+def test_write_excel_bytes() -> None:
+    df = pl.DataFrame(
+        {
+            "A": [1, 2, 3, 4, 5],
+        }
+    )
+    excel_bytes = BytesIO()
+    df.write_excel(excel_bytes)
+    df_read = pl.read_excel(excel_bytes)
+    assert_frame_equal(df, df_read)
+
+
+def test_unsupported_engine() -> None:
+    with pytest.raises(NotImplementedError):
+        pl.read_excel(None, engine="foo")  # type: ignore[call-overload]
+
+
 def test_read_excel_all_sheets_with_sheet_name(excel_file_path: Path) -> None:
     with pytest.raises(
         ValueError,