Skip to content

Commit

Permalink
refactor(pyspark): remove another private function
Browse files Browse the repository at this point in the history
Similar to
#5704
  • Loading branch information
NickCrews authored and cpcloud committed Mar 8, 2023
1 parent e4ea597 commit c5081cf
Show file tree
Hide file tree
Showing 2 changed files with 0 additions and 46 deletions.
24 changes: 0 additions & 24 deletions ibis/backends/pyspark/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -331,30 +331,6 @@ def get_schema(

return sch.infer(df)

def _schema_from_csv(self, path: str, **kwargs: Any) -> sch.Schema:
"""Return a Schema object for the indicated csv file.
Spark goes through the file once to determine the schema.
Parameters
----------
path
Path to CSV
kwargs
See documentation for `pyspark.sql.DataFrameReader` for more information.
Returns
-------
sch.Schema
An ibis schema instance
"""
options = _read_csv_defaults.copy()
options.update(kwargs)
options['inferSchema'] = True

df = self._session.read.csv(path, **options)
return spark_dataframe_schema(df)

def create_table(
self,
table_name: str,
Expand Down
22 changes: 0 additions & 22 deletions ibis/backends/pyspark/tests/test_ddl.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,25 +230,3 @@ def test_create_table_reserved_identifier(client, alltypes):
assert result == expected
finally:
client.drop_table(table_name)


@pytest.fixture(scope='session')
def awards_players_filename(data_directory):
return str(data_directory / 'awards_players.csv')


awards_players_schema = ibis.schema(
[
('playerID', 'string'),
('awardID', 'string'),
('yearID', 'int32'),
('lgID', 'string'),
('tie', 'string'),
('notes', 'string'),
]
)


def test_schema_from_csv(client, awards_players_filename):
schema = client._schema_from_csv(awards_players_filename)
assert schema.equals(awards_players_schema)

0 comments on commit c5081cf

Please sign in to comment.