diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 3ee6e82..b482639 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -4,11 +4,17 @@ - Compatibility with Python 2.7, 3.3, and 3.4 +------------------------------------------------------------------------------ + qPython 1.0.2 [2015.09.03] +------------------------------------------------------------------------------ + + - Enhancement: configuration of serialization for single element strings + ------------------------------------------------------------------------------ qPython 1.0.1 [2015.08.11] ------------------------------------------------------------------------------ - - Fix: serialization of indexed Pandas.DataFrames + - Fix: serialization of indexed Pandas.DataFrames ------------------------------------------------------------------------------ qPython 1.0.0 [2015.04.10] diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 8085194..52deb7b 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: qpython - version: "1.1.0b2" + version: "1.1.0b3" build: number: 1 diff --git a/doc/source/conf.py b/doc/source/conf.py index 560301d..f3600b3 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -70,10 +70,11 @@ def __getattr__(cls, name): # |version| and |release|, also used in various other places throughout the # built documents. # +from qpython import __version__ # The short X.Y version. -version = '1.1' +version = __version__ # The full version, including alpha/beta/rc tags. -release = '1.1.0b2' +release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -261,7 +262,7 @@ def __getattr__(cls, name): # dir menu entry, description, category) texinfo_documents = [ ('index', 'qPython', u'qPython Documentation', - u'DEVnet', 'qPython', 'One line description of project.', + u'DEVnet', 'qPython', 'Interprocess communication between Python and kdb+', 'Miscellaneous'), ] diff --git a/doc/source/connection.rst b/doc/source/connection.rst index caeeacc..0eaf908 100644 --- a/doc/source/connection.rst +++ b/doc/source/connection.rst @@ -30,8 +30,8 @@ can be used with a ``with`` statement: print(q('{`int$ til x}', 10)) -Q parser configuration -********************** +Types conversion configuration +****************************** Connection can be preconfigured to parse IPC messages according to a specified settings, e.g.: temporal vectors can be represented as raw vectors or converted @@ -45,6 +45,7 @@ to numpy `datetime64`/`timedelta64` representation. q = qconnection.QConnection(host = 'localhost', port = 5000, numpy_temporals = True) -Parsing options can be also overwritten while executing synchronous query -(:meth:`~qpython.qconnection.QConnection.sync`) or retrieving data from q +Conversion options can be also overwritten while executing +synchronous/asynchronous queries (:meth:`~qpython.qconnection.QConnection.sync`, +:meth:`~qpython.qconnection.QConnection.async`) or retrieving data from q (:meth:`~qpython.qconnection.QConnection.receive`). \ No newline at end of file diff --git a/doc/source/queries.rst b/doc/source/queries.rst index 160ced3..cf9d3b6 100644 --- a/doc/source/queries.rst +++ b/doc/source/queries.rst @@ -106,15 +106,16 @@ QMessage: message type: 2, data size: 13, is_compressed: False, data: 10 fa0a000000 -Q parser configuration -********************** +Type conversions configuration +****************************** -Parsing options can be overwritten while: +Type conversion options can be overwritten while: - executing synchronous query: :meth:`~qpython.qconnection.QConnection.sync` +- executing asynchronous query: :meth:`~qpython.qconnection.QConnection.async` - retrieving data from q: :meth:`~qpython.qconnection.QConnection.receive` -Both methods accepts the `options` keywords arguments:: +These methods accepts the `options` keywords arguments:: >>> query = "{[x] 0Nd, `date$til x}" @@ -134,4 +135,10 @@ Both methods accepts the `options` keywords arguments:: [NaT [metadata(qtype=-14)] 2000-01-01 [metadata(qtype=-14)] 2000-01-02 [metadata(qtype=-14)] 2000-01-03 [metadata(qtype=-14)]] - \ No newline at end of file + >>> # serialize single element strings as q characters + >>> print(q.sync('{[x] type each x}', ['one', 'two', '3'], single_char_strings = False)) + [ 10, 10, -10] + + >>> # serialize single element strings as q strings + >>> print(q.sync('{[x] type each x}', ['one', 'two', '3'], single_char_strings = True)) + [10, 10, 10] diff --git a/doc/source/type-conversion.rst b/doc/source/type-conversion.rst index fcbd7e4..9011594 100644 --- a/doc/source/type-conversion.rst +++ b/doc/source/type-conversion.rst @@ -22,14 +22,14 @@ to this table: =============== ============ ===================================== q type q num type Python type =============== ============ ===================================== - ``bool`` -1 ``numpy.bool_`` + ``bool`` -1 ``numpy.bool_`` ``guid`` -2 ``UUID`` - ``byte`` -4 ``numpy.byte`` - ``short`` -5 ``numpy.int16`` - ``integer`` -6 ``numpy.int32`` - ``long`` -7 ``numpy.int64`` - ``real`` -8 ``numpy.float32`` - ``float`` -9 ``numpy.float64`` + ``byte`` -4 ``numpy.byte`` + ``short`` -5 ``numpy.int16`` + ``integer`` -6 ``numpy.int32`` + ``long`` -7 ``numpy.int64`` + ``real`` -8 ``numpy.float32`` + ``float`` -9 ``numpy.float64`` ``character`` -10 single element ``str`` ``timestamp`` -12 ``QTemporal numpy.datetime64 ns`` ``month`` -13 ``QTemporal numpy.datetime64 M`` @@ -54,31 +54,35 @@ described in the table: ===================================== ================ ============ Python type q type q num type ===================================== ================ ============ - ``bool`` ``bool`` -1 - --- ``byte`` -4 - --- ``short`` -5 - ``int`` ``int`` -6 - ``long`` ``long`` -7 - --- ``real`` -8 - ``double`` ``float`` -9 - ``numpy.bool`` ``bool`` -1 - ``numpy.byte`` ``byte`` -4 - ``numpy.int16`` ``short`` -5 - ``numpy.int32`` ``int`` -6 - ``numpy.int64`` ``long`` -7 - ``numpy.float32`` ``real`` -8 - ``numpy.float64`` ``float`` -9 - single element ``str`` ``character`` -10 - ``QTemporal numpy.datetime64 ns`` ``timestamp`` -12 - ``QTemporal numpy.datetime64 M`` ``month`` -13 - ``QTemporal numpy.datetime64 D`` ``date`` -14 - ``QTemporal numpy.datetime64 ms`` ``datetime`` -15 - ``QTemporal numpy.timedelta64 ns`` ``timespan`` -16 - ``QTemporal numpy.timedelta64 m`` ``minute`` -17 - ``QTemporal numpy.timedelta64 s`` ``second`` -18 - ``QTemporal numpy.timedelta64 ms`` ``time`` -19 + ``bool`` ``bool`` -1 + --- ``byte`` -4 + --- ``short`` -5 + ``int`` ``int`` -6 + ``long`` ``long`` -7 + --- ``real`` -8 + ``double`` ``float`` -9 + ``numpy.bool`` ``bool`` -1 + ``numpy.byte`` ``byte`` -4 + ``numpy.int16`` ``short`` -5 + ``numpy.int32`` ``int`` -6 + ``numpy.int64`` ``long`` -7 + ``numpy.float32`` ``real`` -8 + ``numpy.float64`` ``float`` -9 + single element ``str`` ``character`` -10 + ``QTemporal numpy.datetime64 ns`` ``timestamp`` -12 + ``QTemporal numpy.datetime64 M`` ``month`` -13 + ``QTemporal numpy.datetime64 D`` ``date`` -14 + ``QTemporal numpy.datetime64 ms`` ``datetime`` -15 + ``QTemporal numpy.timedelta64 ns`` ``timespan`` -16 + ``QTemporal numpy.timedelta64 m`` ``minute`` -17 + ``QTemporal numpy.timedelta64 s`` ``second`` -18 + ``QTemporal numpy.timedelta64 ms`` ``time`` -19 ===================================== ================ ============ +.. note:: By default, single element strings are serialized as q characters. + This setting can be modified (`single_char_strings = True`) and + and single element strings are represented as q strings. + String and symbols ****************** @@ -98,6 +102,20 @@ apply: 'quick brown fox jumps over a lazy dog' +.. note:: By default, single element strings are serialized as q characters. + This setting can be modified (`single_char_strings = True`) and + and single element strings are represented as q strings. + +:: + + >>> # serialize single element strings as q characters + >>> print(q.sync('{[x] type each x}', ['one', 'two', '3'], single_char_strings = False)) + [ 10, 10, -10] + + >>> # serialize single element strings as q strings + >>> print(q.sync('{[x] type each x}', ['one', 'two', '3'], single_char_strings = True)) + [10, 10, 10] + Lists ***** @@ -350,22 +368,22 @@ Complete null mapping between q and Python is represented in the table: ============== ============== ======================= ``bool`` ``0b`` ``_QNULL_BOOL`` ``guid`` ``0Ng`` ``_QNULL_GUID`` - ``byte`` ``0x00`` ``_QNULL1`` - ``short`` ``0Nh`` ``_QNULL2`` - ``int`` ``0N`` ``_QNULL4`` - ``long`` ``0Nj`` ``_QNULL8`` - ``real`` ``0Ne`` ``_QNAN32`` - ``float`` ``0n`` ``_QNAN64`` - ``string`` ``" "`` ``' '`` + ``byte`` ``0x00`` ``_QNULL1`` + ``short`` ``0Nh`` ``_QNULL2`` + ``int`` ``0N`` ``_QNULL4`` + ``long`` ``0Nj`` ``_QNULL8`` + ``real`` ``0Ne`` ``_QNAN32`` + ``float`` ``0n`` ``_QNAN64`` + ``string`` ``" "`` ``' '`` ``symbol`` \` ``_QNULL_SYM`` - ``timestamp`` ``0Np`` ``_QNULL8`` - ``month`` ``0Nm`` ``_QNULL4`` - ``date`` ``0Nd`` ``_QNULL4`` - ``datetime`` ``0Nz`` ``_QNAN64`` - ``timespan`` ``0Nn`` ``_QNULL8`` - ``minute`` ``0Nu`` ``_QNULL4`` - ``second`` ``0Nv`` ``_QNULL4`` - ``time`` ``0Nt`` ``_QNULL4`` + ``timestamp`` ``0Np`` ``_QNULL8`` + ``month`` ``0Nm`` ``_QNULL4`` + ``date`` ``0Nd`` ``_QNULL4`` + ``datetime`` ``0Nz`` ``_QNAN64`` + ``timespan`` ``0Nn`` ``_QNULL8`` + ``minute`` ``0Nu`` ``_QNULL4`` + ``second`` ``0Nv`` ``_QNULL4`` + ``time`` ``0Nt`` ``_QNULL4`` ============== ============== ======================= The :py:mod:`qtype` provides two utility functions to work with null values: diff --git a/qpython/__init__.py b/qpython/__init__.py index d36b409..f945548 100644 --- a/qpython/__init__.py +++ b/qpython/__init__.py @@ -17,7 +17,8 @@ __all__ = ['qconnection', 'qtype', 'qtemporal', 'qcollection'] -__version__ = '1.1.0b2' +__version__ = '1.1.0b3' + try: @@ -58,3 +59,11 @@ def as_dict(self): def union_dict(self, **kw): return dict(list(self.as_dict().items()) + list(kw.items())) + + + +CONVERSION_OPTIONS = MetaData(raw = False, + numpy_temporals = False, + pandas = False, + single_char_strings = False + ) \ No newline at end of file diff --git a/qpython/_pandas.py b/qpython/_pandas.py index 6018db7..e7e127f 100644 --- a/qpython/_pandas.py +++ b/qpython/_pandas.py @@ -23,7 +23,7 @@ from collections import OrderedDict from qpython import MetaData -from qpython.qreader import QReader, READER_CONFIGURATION, QReaderException +from qpython.qreader import QReader, QReaderException from qpython.qcollection import QDictionary, qlist from qpython.qwriter import QWriter, QWriterException from qpython.qtype import * @@ -35,10 +35,10 @@ class PandasQReader(QReader): parse = Mapper(QReader._reader_map) @parse(QDICTIONARY) - def _read_dictionary(self, qtype = QDICTIONARY, options = READER_CONFIGURATION): - if options.pandas: - keys = self._read_object(options = options) - values = self._read_object(options = options) + def _read_dictionary(self, qtype = QDICTIONARY): + if self._options.pandas: + keys = self._read_object() + values = self._read_object() if isinstance(keys, pandas.DataFrame): if not isinstance(values, pandas.DataFrame): @@ -61,17 +61,17 @@ def _read_dictionary(self, qtype = QDICTIONARY, options = READER_CONFIGURATION): values = values if not isinstance(values, pandas.Series) else values.as_matrix() return QDictionary(keys, values) else: - return QReader._read_dictionary(self, qtype = qtype, options = options) + return QReader._read_dictionary(self, qtype = qtype) @parse(QTABLE) - def _read_table(self, qtype = QTABLE, options = READER_CONFIGURATION): - if options.pandas: + def _read_table(self, qtype = QTABLE): + if self._options.pandas: self._buffer.skip() # ignore attributes self._buffer.skip() # ignore dict type stamp - columns = self._read_object(options = options) - data = self._read_object(options = options) + columns = self._read_object() + data = self._read_object() odict = OrderedDict() meta = MetaData(qtype = QTABLE) @@ -99,16 +99,16 @@ def _read_table(self, qtype = QTABLE, options = READER_CONFIGURATION): df.meta = meta return df else: - return QReader._read_table(self, qtype = qtype, options = options) + return QReader._read_table(self, qtype = qtype) - def _read_list(self, qtype, options): - if options.pandas: - options.numpy_temporals = True + def _read_list(self, qtype): + if self._options.pandas: + self._options.numpy_temporals = True - list = QReader._read_list(self, qtype = qtype, options = options) + list = QReader._read_list(self, qtype = qtype) - if options.pandas: + if self._options.pandas: if -abs(qtype) not in [QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN, QSYMBOL]: null = QNULLMAP[-abs(qtype)][1] ps = pandas.Series(data = list).replace(null, numpy.NaN) @@ -122,9 +122,9 @@ def _read_list(self, qtype, options): @parse(QGENERAL_LIST) - def _read_general_list(self, qtype = QGENERAL_LIST, options = READER_CONFIGURATION): - list = QReader._read_general_list(self, qtype, options) - if options.pandas: + def _read_general_list(self, qtype = QGENERAL_LIST): + list = QReader._read_general_list(self, qtype) + if self._options.pandas: return [numpy.nan if isinstance(element, basestring) and element == b' ' else element for element in list] else: return list diff --git a/qpython/qconnection.py b/qpython/qconnection.py index b4a85a9..1fbf447 100644 --- a/qpython/qconnection.py +++ b/qpython/qconnection.py @@ -17,14 +17,15 @@ import socket import struct -from qpython import MetaData +from qpython import MetaData, CONVERSION_OPTIONS from qpython.qtype import QException -from qpython.qreader import QReader, QReaderException, READER_CONFIGURATION +from qpython.qreader import QReader, QReaderException from qpython.qwriter import QWriter, QWriterException + class QConnectionException(Exception): - '''Raised when a connection to the q service cannot be estabilished.''' + '''Raised when a connection to the q service cannot be established.''' pass @@ -69,6 +70,8 @@ class QConnection(object): :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` + - `single_char_strings` (`boolean`) - if ``True`` single char Python + strings are encoded as q strings instead of chars, **Default**: ``False`` ''' def __init__(self, host, port, username = None, password = None, timeout = None, **options): @@ -82,7 +85,7 @@ def __init__(self, host, port, username = None, password = None, timeout = None, self.timeout = timeout - self._options = MetaData(**READER_CONFIGURATION.union_dict(**options)) + self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) def __enter__(self): @@ -178,7 +181,7 @@ def __str__(self): return '%s@:%s:%s' % (self.username, self.host, self.port) if self.username else ':%s:%s' % (self.host, self.port) - def query(self, msg_type, query, *parameters): + def query(self, msg_type, query, *parameters, **options): '''Performs a query against a q service. In typical use case, `query` is the name of the function to call and @@ -198,6 +201,10 @@ def query(self, msg_type, query, *parameters): type of the query to be executed - `query` (`string`) - query to be executed - `parameters` (`list` or `None`) - parameters for the query + :Options: + - `single_char_strings` (`boolean`) - if ``True`` single char Python + strings are encoded as q strings instead of chars, + **Default**: ``False`` :raises: :class:`.QConnectionException`, :class:`.QWriterException` ''' @@ -208,9 +215,9 @@ def query(self, msg_type, query, *parameters): raise QWriterException('Too many parameters.') if not parameters or len(parameters) == 0: - self._writer.write(query, msg_type) + self._writer.write(query, msg_type, **self._options.union_dict(**options)) else: - self._writer.write([query] + list(parameters), msg_type) + self._writer.write([query] + list(parameters), msg_type, **self._options.union_dict(**options)) def sync(self, query, *parameters, **options): @@ -258,13 +265,16 @@ def sync(self, query, *parameters, **options): :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` + - `single_char_strings` (`boolean`) - if ``True`` single char Python + strings are encoded as q strings instead of chars, + **Default**: ``False`` :returns: query result parsed to Python data structures :raises: :class:`.QConnectionException`, :class:`.QWriterException`, :class:`.QReaderException` ''' - self.query(MessageType.SYNC, query, *parameters) + self.query(MessageType.SYNC, query, *parameters, **options) response = self.receive(data_only = False, **options) if response.type == MessageType.RESPONSE: @@ -274,7 +284,7 @@ def sync(self, query, *parameters, **options): raise QReaderException('Received message of type: %s where response was expected') - def async(self, query, *parameters): + def async(self, query, *parameters, **options): '''Performs an asynchronous query and returns **without** retrieving of the response. @@ -293,10 +303,14 @@ def async(self, query, *parameters): :Parameters: - `query` (`string`) - query to be executed - `parameters` (`list` or `None`) - parameters for the query + :Options: + - `single_char_strings` (`boolean`) - if ``True`` single char Python + strings are encoded as q strings instead of chars, + **Default**: ``False`` :raises: :class:`.QConnectionException`, :class:`.QWriterException` ''' - self.query(MessageType.ASYNC, query, *parameters) + self.query(MessageType.ASYNC, query, *parameters, **options) def receive(self, data_only = True, **options): diff --git a/qpython/qreader.py b/qpython/qreader.py index e7aa90a..bac1e92 100644 --- a/qpython/qreader.py +++ b/qpython/qreader.py @@ -20,7 +20,7 @@ from sys import intern unicode = str -from qpython import MetaData +from qpython import MetaData, CONVERSION_OPTIONS from qpython.qtype import * # @UnusedWildImport from qpython.qcollection import qlist, QDictionary, qtable, QTable, QKeyedTable from qpython.qtemporal import qtemporal, from_raw_qtemporal, array_from_raw_qtemporal @@ -32,12 +32,6 @@ -READER_CONFIGURATION = MetaData(raw = False, - numpy_temporals = False, - pandas = False) - - - class QReaderException(Exception): ''' Indicates an error raised during data deserialization. @@ -204,7 +198,7 @@ def read_data(self, message_size, is_compressed = False, **options): :returns: read data (parsed or raw byte form) ''' - options = MetaData(**READER_CONFIGURATION.union_dict(**options)) + self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) if is_compressed: if self._stream: @@ -222,55 +216,55 @@ def read_data(self, message_size, is_compressed = False, **options): elif self._stream: raw_data = self._read_bytes(message_size - 8) self._buffer.wrap(raw_data) - if not self._stream and options.raw: + if not self._stream and self._options.raw: raw_data = self._buffer.raw(message_size - 8) - return raw_data if options.raw else self._read_object(options) + return raw_data if self._options.raw else self._read_object() - def _read_object(self, options = READER_CONFIGURATION): + def _read_object(self): qtype = self._buffer.get_byte() reader = QReader._reader_map.get(qtype, None) if reader: - return reader(self, qtype, options) + return reader(self, qtype) elif qtype >= QBOOL_LIST and qtype <= QTIME_LIST: - return self._read_list(qtype, options) + return self._read_list(qtype) elif qtype <= QBOOL and qtype >= QTIME: - return self._read_atom(qtype, options) + return self._read_atom(qtype) raise QReaderException('Unable to deserialize q type: %s' % hex(qtype)) @parse(QERROR) - def _read_error(self, qtype = QERROR, options = READER_CONFIGURATION): - raise QException(self._read_symbol(options = options)) + def _read_error(self, qtype = QERROR): + raise QException(self._read_symbol()) @parse(QSTRING) - def _read_string(self, qtype = QSTRING, options = READER_CONFIGURATION): + def _read_string(self, qtype = QSTRING): self._buffer.skip() # ignore attributes length = self._buffer.get_int() return self._buffer.raw(length) if length > 0 else b'' @parse(QSYMBOL) - def _read_symbol(self, qtype = QSYMBOL, options = READER_CONFIGURATION): + def _read_symbol(self, qtype = QSYMBOL): return numpy.string_(self._buffer.get_symbol()) @parse(QCHAR) - def _read_char(self, qtype = QCHAR, options = READER_CONFIGURATION): - return chr(self._read_atom(QCHAR, options)).encode('latin-1') + def _read_char(self, qtype = QCHAR): + return chr(self._read_atom(QCHAR)).encode('latin-1') @parse(QGUID) - def _read_guid(self, qtype = QGUID, options = READER_CONFIGURATION): + def _read_guid(self, qtype = QGUID): return uuid.UUID(bytes = self._buffer.raw(16)) - def _read_atom(self, qtype, options): + def _read_atom(self, qtype): try: fmt = STRUCT_MAP[qtype] conversion = PY_TYPE[qtype] @@ -280,17 +274,17 @@ def _read_atom(self, qtype, options): @parse(QTIMESPAN, QTIMESTAMP, QTIME, QSECOND, QMINUTE, QDATE, QMONTH, QDATETIME) - def _read_temporal(self, qtype, options): + def _read_temporal(self, qtype): try: fmt = STRUCT_MAP[qtype] conversion = PY_TYPE[qtype] temporal = from_raw_qtemporal(conversion(self._buffer.get(fmt)), qtype = qtype) - return temporal if options.numpy_temporals else qtemporal(temporal, qtype = qtype) + return temporal if self._options.numpy_temporals else qtemporal(temporal, qtype = qtype) except KeyError: raise QReaderException('Unable to deserialize q type: %s' % hex(qtype)) - def _read_list(self, qtype, options): + def _read_list(self, qtype): self._buffer.skip() # ignore attributes length = self._buffer.get_int() conversion = PY_TYPE.get(-qtype, None) @@ -308,7 +302,7 @@ def _read_list(self, qtype, options): if not self._is_native: data.byteswap(True) - if qtype >= QTIMESTAMP_LIST and qtype <= QTIME_LIST and options.numpy_temporals: + if qtype >= QTIMESTAMP_LIST and qtype <= QTIME_LIST and self._options.numpy_temporals: data = array_from_raw_qtemporal(data, qtype) return qlist(data, qtype = qtype, adjust_dtype = False) @@ -317,9 +311,9 @@ def _read_list(self, qtype, options): @parse(QDICTIONARY) - def _read_dictionary(self, qtype = QDICTIONARY, options = READER_CONFIGURATION): - keys = self._read_object(options = options) - values = self._read_object(options = options) + def _read_dictionary(self, qtype = QDICTIONARY): + keys = self._read_object() + values = self._read_object() if isinstance(keys, QTable): return QKeyedTable(keys, values) @@ -328,42 +322,42 @@ def _read_dictionary(self, qtype = QDICTIONARY, options = READER_CONFIGURATION): @parse(QTABLE) - def _read_table(self, qtype = QTABLE, options = READER_CONFIGURATION): + def _read_table(self, qtype = QTABLE): self._buffer.skip() # ignore attributes self._buffer.skip() # ignore dict type stamp - columns = self._read_object(options = options) - data = self._read_object(options = options) + columns = self._read_object() + data = self._read_object() return qtable(columns, data, qtype = QTABLE) @parse(QGENERAL_LIST) - def _read_general_list(self, qtype = QGENERAL_LIST, options = READER_CONFIGURATION): + def _read_general_list(self, qtype = QGENERAL_LIST): self._buffer.skip() # ignore attributes length = self._buffer.get_int() - return [self._read_object(options = options) for x in range(length)] + return [self._read_object() for x in range(length)] @parse(QNULL) @parse(QUNARY_FUNC) @parse(QBINARY_FUNC) @parse(QTERNARY_FUNC) - def _read_function(self, qtype = QNULL, options = None): + def _read_function(self, qtype = QNULL): code = self._buffer.get_byte() return None if qtype == QNULL and code == 0 else QFunction(qtype) @parse(QLAMBDA) - def _read_lambda(self, qtype = QLAMBDA, options = READER_CONFIGURATION): + def _read_lambda(self, qtype = QLAMBDA): self._buffer.get_symbol() # skip - expression = self._read_object(options = options) + expression = self._read_object() return QLambda(expression.decode()) @parse(QCOMPOSITION_FUNC) - def _read_function_composition(self, qtype = QCOMPOSITION_FUNC, options = None): + def _read_function_composition(self, qtype = QCOMPOSITION_FUNC): self._read_projection(qtype) # skip return QFunction(qtype) @@ -374,15 +368,15 @@ def _read_function_composition(self, qtype = QCOMPOSITION_FUNC, options = None): @parse(QADVERB_FUNC_109) @parse(QADVERB_FUNC_110) @parse(QADVERB_FUNC_111) - def _read_adverb_function(self, qtype = QADVERB_FUNC_106, options = None): + def _read_adverb_function(self, qtype = QADVERB_FUNC_106): self._read_object() # skip return QFunction(qtype) @parse(QPROJECTION) - def _read_projection(self, qtype = QPROJECTION, options = READER_CONFIGURATION): + def _read_projection(self, qtype = QPROJECTION): length = self._buffer.get_int() - parameters = [ self._read_object(options = options) for x in range(length) ] + parameters = [ self._read_object() for x in range(length) ] return QProjection(parameters) @@ -510,9 +504,9 @@ def get_int(self): def get_symbol(self): ''' - Gets a single, ``\x00`` terminated string from the buffer. + Gets a single, ``\\x00`` terminated string from the buffer. - :returns: ``\x00`` terminated string + :returns: ``\\x00`` terminated string ''' new_position = self._data.find(b'\x00', self._position) @@ -526,12 +520,12 @@ def get_symbol(self): def get_symbols(self, count): ''' - Gets ``count`` ``\x00`` terminated strings from the buffer. + Gets ``count`` ``\\x00`` terminated strings from the buffer. :Parameters: - `count` (`integer`) - number of strings to be read - :returns: list of ``\x00`` terminated string read from the buffer + :returns: list of ``\\x00`` terminated string read from the buffer ''' c = 0 new_position = self._position diff --git a/qpython/qwriter.py b/qpython/qwriter.py index 3a6cc80..e4a2d0a 100644 --- a/qpython/qwriter.py +++ b/qpython/qwriter.py @@ -20,6 +20,7 @@ except ImportError: from io import BytesIO +from qpython import MetaData, CONVERSION_OPTIONS from qpython.qtype import * # @UnusedWildImport from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, QTable, QKeyedTable, get_list_qtype from qpython.qtemporal import QTemporal, to_raw_qtemporal, array_to_raw_qtemporal @@ -66,19 +67,25 @@ def __init__(self, stream, protocol_version): self._protocol_version = protocol_version - def write(self, data, msg_type): + def write(self, data, msg_type, **options): '''Serializes and pushes single data object to a wrapped stream. :Parameters: - `data` - data to be serialized - `msg_type` (one of the constants defined in :class:`.MessageType`) - type of the message + :Options: + - `single_char_strings` (`boolean`) - if ``True`` single char Python + strings are encoded as q strings instead of chars, + **Default**: ``False`` :returns: if wraped stream is ``None`` serialized data, otherwise ``None`` ''' self._buffer = BytesIO() + self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) + # header and placeholder for message size self._buffer.write(('%s%s\0\0\0\0\0\0' % (ENDIANESS, chr(msg_type))).encode("latin-1")) @@ -154,7 +161,7 @@ def _write_generic_list(self, data): @serialize(str, bytes) def _write_string(self, data): - if len(data) == 1: + if not self._options.single_char_strings and len(data) == 1: self._write_atom(ord(data), QCHAR) else: self._buffer.write(struct.pack('=bxi', QSTRING, len(data))) diff --git a/setup.py b/setup.py index 43b6e8c..122f2cc 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ # from distutils.core import setup +from qpython import __version__ import numpy import os @@ -38,7 +39,7 @@ def read(fname): setup(name = 'qPython', - version = '1.1.0b2', + version = __version__, description = 'kdb+ interfacing library for Python', long_description=read('README.rst'), diff --git a/tests/QExpressions3.out b/tests/QExpressions3.out index e69ef29..30653bf 100644 --- a/tests/QExpressions3.out +++ b/tests/QExpressions3.out @@ -134,6 +134,10 @@ ED00000080 0000070000000A0005000000717569636B0A000500000062726F776E0A0003000000666F780A00050000006A756D70730A00040000006F7665720A000600000061206C617A790A0003000000646F67 ("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog") 0000070000000A0005000000717569636BF6200A0003000000666F780A00050000006A756D70730A00040000006F7665720A000600000061206C617A790A0003000000646F67 +("one"; "two"; "3") +0000030000000a00030000006f6e650a000300000074776ff633 +("one"; "two"; enlist "3") +0000030000000a00030000006f6e650a000300000074776f0a000100000033 2000.01.04D05:36:57.600 0Np 0C000200000000C0CAFA20FE00000000000000000080 (2001.01m; 0Nm) diff --git a/tests/qwriter_test.py b/tests/qwriter_test.py index 3fe197f..9e84503 100644 --- a/tests/qwriter_test.py +++ b/tests/qwriter_test.py @@ -348,9 +348,9 @@ def test_writing(): for query, value in iter(EXPRESSIONS.items()): sys.stdout.write( '%-75s' % query ) if isinstance(value, tuple): - for object in value: + for obj in value: sys.stdout.write( '.' ) - serialized = binascii.hexlify(w.write(object, 1))[16:].lower() + serialized = binascii.hexlify(w.write(obj, 1))[16:].lower() assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized) else: sys.stdout.write( '.' ) @@ -359,7 +359,17 @@ def test_writing(): print('') +def test_write_single_char_string(): + w = qwriter.QWriter(None, 3) + + for obj in (['one', 'two', '3'], qlist(['one', 'two', '3'], qtype = QSTRING_LIST)): + single_char_strings = False + for query in (b'("one"; "two"; "3")', b'("one"; "two"; enlist "3")'): + serialized = binascii.hexlify(w.write(obj, 1, single_char_strings = single_char_strings ))[16:].lower() + assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized) + single_char_strings = not single_char_strings init() -test_writing() \ No newline at end of file +test_writing() +test_write_single_char_string() \ No newline at end of file