Skip to content

Commit

Permalink
Merge branch 'maciejlach-master'
Browse files Browse the repository at this point in the history
  • Loading branch information
@joannajarmulska
joannajarmulska committed Feb 2, 2017
2 parents 2ac86a0 + ce020e7 commit 0010f79
Show file tree
Hide file tree
Showing 7 changed files with 175 additions and 104 deletions.
14 changes: 10 additions & 4 deletions CHANGELOG.txt
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
------------------------------------------------------------------------------
qPython 1.2.2 [2016.09.21]
------------------------------------------------------------------------------

- Fix: Pandas deserialization of single row tables with string nulls

------------------------------------------------------------------------------
qPython 1.2.1 [2016.03.29]
------------------------------------------------------------------------------
Expand Down Expand Up @@ -32,13 +38,13 @@
qPython 1.0.0 [2015.04.10]
------------------------------------------------------------------------------

- Improve type detection for pandas.Series serialization
- Improve type detection for pandas.Series serialization

------------------------------------------------------------------------------
qPython 1.0 RC3 [2015.04.09]
------------------------------------------------------------------------------

- Fix: bug during handshake with blank username/password
- Fix: bug during handshake with blank username/password

------------------------------------------------------------------------------
qPython 1.0 RC2 [2015.01.19]
Expand All @@ -58,7 +64,7 @@
------------------------------------------------------------------------------

- Configurable deserialization for temporal vectors
- numpy datetime64/timedelta64 vectors ans atoms are now serializable without
- numpy datetime64/timedelta64 vectors ans atoms are now serializable without
explicit conversions
- QTemporal: null values are now backed as numpy NaT
- Fix: error handling while initializing connection
Expand All @@ -79,7 +85,7 @@
qPython 1.0 Beta 4 [2014.07.04]
------------------------------------------------------------------------------

- qtemporallist: force numpy.array type conversion in case of mismatch between
- qtemporallist: force numpy.array type conversion in case of mismatch between
meta.qtype and dtype of raw list
- Enable Travis CI integration
- Update project meta-information
Expand Down
4 changes: 2 additions & 2 deletions qpython/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
__all__ = ['qconnection', 'qtype', 'qtemporal', 'qcollection']


__version__ = '1.2.1'
__version__ = '1.2.2'



Expand Down Expand Up @@ -66,4 +66,4 @@ def union_dict(self, **kw):
numpy_temporals = False,
pandas = False,
single_char_strings = False
)
)
3 changes: 2 additions & 1 deletion qpython/_pandas.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ def _read_table(self, qtype = QTABLE):
self._buffer.skip() # ignore dict type stamp

columns = self._read_object()
data = self._read_object()
self._buffer.skip() # ignore generic list type indicator
data = QReader._read_general_list(self, qtype)

odict = OrderedDict()
meta = MetaData(qtype = QTABLE)
Expand Down
10 changes: 9 additions & 1 deletion tests/QExpressions3.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,4 +225,12 @@ FE8C680A015A495AAB5A65D4BFDDB6A661
"G"$"00000000-0000-0000-0000-000000000000"
FE00000000000000000000000000000000
("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)
0200020000008C680A015A495AAB5A65D4BFDDB6A66100000000000000000000000000000000
0200020000008C680A015A495AAB5A65D4BFDDB6A66100000000000000000000000000000000
1#([] sym:`x`x`x;str:" a")
6200630b000200000073796d00737472000000020000000b000100000078000a000100000020
-1#([] sym:`x`x`x;str:" a")
6200630b000200000073796d00737472000000020000000b000100000078000a000100000061
2#([] sym:`x`x`x`x;str:" aa")
6200630b000200000073796d00737472000000020000000b0002000000780078000a00020000002020
-2#([] sym:`x`x`x`x;str:" aa")
6200630b000200000073796d00737472000000020000000b0002000000780078000a00020000006161
45 changes: 34 additions & 11 deletions tests/pandas_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
try:
import pandas
BINARY = None

PANDAS_EXPRESSIONS = OrderedDict((
(b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
{'data': pandas.Series(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), numpy.nan])),
Expand All @@ -49,7 +49,7 @@
[b'quick', b'brown', b'fox', b'jumps', b'over', b'a lazy', b'dog']),
(b'("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")',
[b'quick', numpy.nan, b'fox', b'jumps', b'over', b'a lazy', b'dog']),

(b'(0b;1b;0b)', {'data': pandas.Series(numpy.array([False, True, False], dtype = numpy.bool)),
'meta': MetaData(qtype = QBOOL_LIST) }),
(b'(0x01;0x02;0xff)', {'data': pandas.Series(numpy.array([1, 2, 0xff], dtype = numpy.int8)),
Expand Down Expand Up @@ -78,7 +78,7 @@
'meta': MetaData(qtype = QDOUBLE_LIST) }),
(b'3.23 0n', {'data': pandas.Series(numpy.array([3.23, numpy.nan])),
'meta': MetaData(qtype = QDOUBLE_LIST) }),

(b'(2001.01m; 0Nm)', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]')),
'meta': MetaData(qtype = QMONTH_LIST) }),
(b'2001.01.01 2000.05.01 0Nd', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')),
Expand All @@ -95,7 +95,7 @@
'meta': MetaData(qtype = QTIMESTAMP_LIST) }),
(b'0D05:36:57.600 0Nn', {'data': pandas.Series(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')])),
'meta': MetaData(qtype = QTIMESPAN_LIST) }),

(b'1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))),
(b'(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)],
Expand All @@ -121,6 +121,32 @@
))
),
'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QLONG_LIST, 'grade': QSTRING}) }),
(b'1#([] sym:`x`x`x;str:" a")',
{'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x'], dtype = numpy.string_)),
('str', pandas.Series([' '], dtype = numpy.str).replace(b' ', numpy.nan)),
))
),
'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING}),
'single_char_strings': True}),
(b'-1#([] sym:`x`x`x;str:" a")',
{'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x'], dtype = numpy.string_)),
('str', pandas.Series(['a'], dtype = numpy.str)),
))
),
'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING}),
'single_char_strings': True}),
(b'2#([] sym:`x`x`x`x;str:" aa")',
{'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x', 'x'], dtype = numpy.string_)),
('str', pandas.Series([' ', ' '], dtype = numpy.str).replace(b' ', numpy.nan)),
))
),
'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING})}),
(b'-2#([] sym:`x`x`x`x;str:" aa")',
{'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x', 'x'], dtype = numpy.string_)),
('str', pandas.Series(['a', 'a'], dtype = numpy.str).replace(b' ', numpy.nan)),
))
),
'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING})}),
(b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))',
{'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'], dtype = numpy.string_)),
('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64))),
Expand All @@ -147,7 +173,6 @@
('iq', pandas.Series(numpy.array([], dtype = numpy.int32)))))
),
'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QINT_LIST}) }),

(b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
{'data': pandas.DataFrame(OrderedDict((('pos', pandas.Series(numpy.array(['d1', 'd2', 'd3'], dtype = numpy.string_))),
('dates', pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')))))
Expand Down Expand Up @@ -276,15 +301,18 @@ def test_writing_pandas():

for query, value in iter(PANDAS_EXPRESSIONS.items()):
sys.stdout.write( '%-75s' % query )
single_char_strings = False
if isinstance(value, dict):
data = value['data']
if 'index' in value:
data = data.reset_index(drop = True)
data = data.set_index(value['index'])
if 'single_char_strings' in value:
single_char_strings = value['single_char_strings']
data.meta = value['meta']
else:
data = value
serialized = binascii.hexlify(w.write(data, 1))[16:].lower()
serialized = binascii.hexlify(w.write(data, 1, single_char_strings = single_char_strings))[16:].lower()
assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (value, BINARY[query].lower(), serialized)
sys.stdout.write( '.' )

Expand Down Expand Up @@ -312,8 +340,3 @@ def test_writing_pandas():
test_writing_pandas()
except ImportError:
pandas = None





Loading

0 comments on commit 0010f79

Please sign in to comment.