Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

FIX-#2239: Compute row index start using pandas #2240

Merged
merged 3 commits into from
Oct 27, 2020
Merged

FIX-#2239: Compute row index start using pandas #2240

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
23d7e3c
FIX-#2239: Compute row index start using pandas
devin-petersohn Oct 13, 2020
0d426b6
FIX-#2239: Documentation
devin-petersohn Oct 13, 2020
d43fce0
FIX-#2239: Improve testing for case
devin-petersohn Oct 27, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions modin/engines/base/io/text/csv_reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,11 +180,6 @@ def _read(cls, filepath_or_buffer, **kwargs):
if index_col is None:
row_lengths = cls.materialize(index_ids)
new_index = pandas.RangeIndex(sum(row_lengths))
# pandas has a really weird edge case here.
if kwargs.get("names", None) is not None and skiprows > 1:
new_index = pandas.RangeIndex(
skiprows - 1, new_index.stop + skiprows - 1
)
else:
index_objs = cls.materialize(index_ids)
row_lengths = [len(o) for o in index_objs]
Expand Down
146 changes: 146 additions & 0 deletions modin/pandas/test/data/issue_2239.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
1585542839.000000, 1585542839.000000, 1585542839.000000
32.000000, 32.000000, 32.000000
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
-38,-14,51
-38,-13,51
-38,-14,51
-38,-14,50
-38,-13,51
-38,-14,50
-38,-14,51
-38,-13,51
27 changes: 26 additions & 1 deletion modin/pandas/test/test_io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1113,7 +1113,7 @@ def test_from_csv_chunksize(make_csv_file):
df_equals(modin_df, pd_df)


@pytest.mark.parametrize("nrows", [123, None])
@pytest.mark.parametrize("nrows", [1, 2, 123, None])
def test_from_csv_skiprows(make_csv_file, nrows):
make_csv_file()

Expand All @@ -1129,6 +1129,22 @@ def test_from_csv_skiprows(make_csv_file, nrows):
)
df_equals(modin_df, pandas_df)

pandas_df = pandas.read_csv(
TEST_CSV_FILENAME,
header=None,
names=["c1", "c2", "c3", "c4"],
skiprows=2,
nrows=nrows,
)
modin_df = pd.read_csv(
TEST_CSV_FILENAME,
header=None,
names=["c1", "c2", "c3", "c4"],
skiprows=2,
nrows=nrows,
)
df_equals(modin_df, pandas_df)

pandas_df = pandas.read_csv(
TEST_CSV_FILENAME,
names=["c1", "c2", "c3", "c4"],
Expand All @@ -1144,6 +1160,15 @@ def test_from_csv_skiprows(make_csv_file, nrows):
df_equals(modin_df, pandas_df)


@pytest.mark.parametrize("names", [list("XYZ"), None])
@pytest.mark.parametrize("skiprows", [1, 2, 3, 4, None])
def test_from_csv_skiprows_names(names, skiprows):
path = "modin/pandas/test/data/issue_2239.csv"
pandas_df = pandas.read_csv(path, names=names, skiprows=skiprows)
modin_df = pd.read_csv(path, names=names, skiprows=skiprows)
df_equals(pandas_df, modin_df)


@pytest.mark.parametrize(
"encoding", ["latin8", "ISO-8859-1", "latin1", "iso-8859-1", "cp1252", "utf8"]
)
Expand Down