From a191744047bcde4e4479ede3f63995d5925ba6ed Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Thu, 11 Aug 2022 07:37:40 -0400 Subject: [PATCH] feat(pyspark): implement `ibis.connect` for pyspark --- ibis/backends/pyspark/__init__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ibis/backends/pyspark/__init__.py b/ibis/backends/pyspark/__init__.py index 754c0c45d8c5..f344853f2a1b 100644 --- a/ibis/backends/pyspark/__init__.py +++ b/ibis/backends/pyspark/__init__.py @@ -1,18 +1,16 @@ from __future__ import annotations +import itertools from pathlib import Path from typing import TYPE_CHECKING, Any, Mapping import pandas as pd import pyspark +import sqlalchemy as sa from pydantic import Field -from pyspark.sql import DataFrame +from pyspark.sql import DataFrame, SparkSession from pyspark.sql.column import Column -if TYPE_CHECKING: - import ibis.expr.types as ir - import ibis.expr.operations as ops - import ibis.common.exceptions as com import ibis.config import ibis.expr.schema as sch @@ -35,6 +33,10 @@ from ibis.expr.scope import Scope from ibis.expr.timecontext import canonicalize_context, localize_context +if TYPE_CHECKING: + import ibis.expr.operations as ops + import ibis.expr.types as ir + _read_csv_defaults = { 'header': True, 'multiLine': True, @@ -119,8 +121,8 @@ def do_connect(self, session: SparkSession) -> None: Examples -------- >>> import ibis - >>> import pyspark - >>> session = pyspark.sql.SparkSession.builder.getOrCreate() + >>> from pyspark.sql import SparkSession + >>> session = SparkSession.builder.getOrCreate() >>> ibis.pyspark.connect(session) """