Skip to content

Commit

Permalink
Support writing CSV to GCS (#22704)
Browse files Browse the repository at this point in the history
  • Loading branch information
bnaul authored and WillAyd committed Oct 12, 2018
1 parent e4b67ca commit 241bde1
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 4 deletions.
2 changes: 1 addition & 1 deletion doc/source/whatsnew/v0.24.0.txt
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ Other Enhancements
- :func:`to_csv` now supports ``compression`` keyword when a file handle is passed. (:issue:`21227`)
- :meth:`Index.droplevel` is now implemented also for flat indexes, for compatibility with :class:`MultiIndex` (:issue:`21115`)
- :meth:`Series.droplevel` and :meth:`DataFrame.droplevel` are now implemented (:issue:`20342`)
- Added support for reading from Google Cloud Storage via the ``gcsfs`` library (:issue:`19454`)
- Added support for reading from/writing to Google Cloud Storage via the ``gcsfs`` library (:issue:`19454`, :issue:`23094`)
- :func:`to_gbq` and :func:`read_gbq` signature and documentation updated to
reflect changes from the `Pandas-GBQ library version 0.6.0
<https://pandas-gbq.readthedocs.io/en/latest/changelog.html#changelog-0-6-0>`__.
Expand Down
7 changes: 4 additions & 3 deletions pandas/io/formats/csvs.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,9 @@
ABCMultiIndex, ABCPeriodIndex, ABCDatetimeIndex, ABCIndexClass)

from pandas.io.common import (
_expand_user,
_get_handle,
_infer_compression,
_stringify_path,
get_filepath_or_buffer,
UnicodeWriter,
)

Expand All @@ -45,7 +44,9 @@ def __init__(self, obj, path_or_buf=None, sep=",", na_rep='',
if path_or_buf is None:
path_or_buf = StringIO()

self.path_or_buf = _expand_user(_stringify_path(path_or_buf))
self.path_or_buf, _, _, _ = get_filepath_or_buffer(
path_or_buf, encoding=encoding, compression=compression, mode=mode
)
self.sep = sep
self.na_rep = na_rep
self.float_format = float_format
Expand Down
15 changes: 15 additions & 0 deletions pandas/tests/io/test_gcs.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,21 @@ def test_read_csv_gcs(mock):
assert_frame_equal(df1, df2)


@td.skip_if_no('gcsfs')
def test_to_csv_gcs(mock):
df1 = DataFrame({'int': [1, 3], 'float': [2.0, np.nan], 'str': ['t', 's'],
'dt': date_range('2018-06-18', periods=2)})
with mock.patch('gcsfs.GCSFileSystem') as MockFileSystem:
s = StringIO()
instance = MockFileSystem.return_value
instance.open.return_value = s

df1.to_csv('gs://test/test.csv', index=True)
df2 = read_csv(StringIO(s.getvalue()), parse_dates=['dt'], index_col=0)

assert_frame_equal(df1, df2)


@td.skip_if_no('gcsfs')
def test_gcs_get_filepath_or_buffer(mock):
df1 = DataFrame({'int': [1, 3], 'float': [2.0, np.nan], 'str': ['t', 's'],
Expand Down

0 comments on commit 241bde1

Please sign in to comment.