From 00bd374dc3320871458ed1a19e7a33882906bfd4 Mon Sep 17 00:00:00 2001 From: Maciej Lach Date: Tue, 4 Aug 2015 15:56:31 +0200 Subject: [PATCH] Documentation amendments --- doc/source/conf.py | 6 +- doc/source/connection.rst | 4 +- doc/source/pandas.rst | 16 +- doc/source/queries.rst | 27 +-- doc/source/type-conversion.rst | 8 +- doc/source/usage-examples.rst | 298 +++++++++++++++++---------------- samples/console.py | 1 - 7 files changed, 189 insertions(+), 171 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index e3e2674..e3576ba 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -64,16 +64,16 @@ def __getattr__(cls, name): # General information about the project. project = u'qPython' -copyright = u'2014, DEVnet' +copyright = u'2014-2015, DEVnet' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '1.0' +version = '1.1' # The full version, including alpha/beta/rc tags. -release = '1.0-beta' +release = '1.1.0b1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/connection.rst b/doc/source/connection.rst index 9f63417..caeeacc 100644 --- a/doc/source/connection.rst +++ b/doc/source/connection.rst @@ -26,8 +26,8 @@ can be used with a ``with`` statement: :: with qconnection.QConnection(host = 'localhost', port = 5000) as q: - print q - print q('{`int$ til x}', 10) + print(q) + print(q('{`int$ til x}', 10)) Q parser configuration diff --git a/doc/source/pandas.rst b/doc/source/pandas.rst index 6aac61a..7387366 100644 --- a/doc/source/pandas.rst +++ b/doc/source/pandas.rst @@ -17,35 +17,35 @@ For example: >>> with qconnection.QConnection(host = 'localhost', port = 5000, pandas = True) as q: >>> ds = q('(1i;0Ni;3i)', pandas = True) - >>> print ds + >>> print(ds) 0 1 1 NaN 2 3 dtype: float64 - >>> print ds.meta + >>> print(ds.meta) metadata(qtype=6) >>> df = q('flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))') - >>> print df + >>> print(df) name iq fullname 0 Dent 98 Arthur Dent 1 Beeblebrox 42 Zaphod Beeblebrox 2 Prefect 126 Ford Prefect - >>> print df.meta + >>> print(df.meta) metadata(iq=7, fullname=0, qtype=98, name=11) - >>> print q('type', df) + >>> print(q('type', df)) 98 >>> df = q('([eid:1001 0N 1003;sym:`foo`bar`] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))') - >>> print df + >>> print(df) pos dates eid sym 1001 foo d1 2001-01-01 NaN bar d2 2000-05-01 1003 d3 NaT - >>> print df.meta + >>> print(df.meta) metadata(dates=14, qtype=99, eid=7, sym=11, pos=11) - >>> print q('type', df) + >>> print(q('type', df)) 99 diff --git a/doc/source/queries.rst b/doc/source/queries.rst index f4d0b9b..160ced3 100644 --- a/doc/source/queries.rst +++ b/doc/source/queries.rst @@ -35,26 +35,26 @@ Synchronous queries Executes a q expression: - >>> print q.sync('til 10') + >>> print(q.sync('til 10')) [0 1 2 3 4 5 6 7 8 9] Executes an anonymous q function with a single parameter: - >>> print q.sync('{til x}', 10) + >>> print(q.sync('{til x}', 10)) [0 1 2 3 4 5 6 7 8 9] Executes an anonymous q function with two parameters: - >>> print q.sync('{y + til x}', 10, 1) + >>> print(q.sync('{y + til x}', 10, 1)) [ 1 2 3 4 5 6 7 8 9 10] - >>> print q.sync('{y + til x}', *[10, 1]) + >>> print(q.sync('{y + til x}', *[10, 1])) [ 1 2 3 4 5 6 7 8 9 10] The :class:`.QConnection` class implements the :func:`~qpython.qconnection.QConnection.__call__` method. This allows :class:`.QConnection` instance to be called as a function: - >>> print q('{y + til x}', 10, 1) + >>> print(q('{y + til x}', 10, 1)) [ 1 2 3 4 5 6 7 8 9 10] @@ -84,25 +84,25 @@ For example: - Retrieves query result along with meta-information: >>> q.query(qconnection.MessageType.SYNC,'{x}', 10) ->>> print q.receive(data_only = False, raw = False) +>>> print(q.receive(data_only = False, raw = False)) QMessage: message type: 2, data size: 13, is_compressed: False, data: 10 - Retrieves parsed query result: >>> q.query(qconnection.MessageType.SYNC,'{x}', 10) ->>> print q.receive(data_only = True, raw = False) +>>> print(q.receive(data_only = True, raw = False)) 10 ->>> q.sync('asynchMult:{[a;b] res:a*b; (neg .z.w)(res) }'); +>>> q.sync('asynchMult:{[a;b] res:a*b; (neg .z.w)(res) }') >>> q.async('asynchMult', 2, 3) ->>> print q.receive() +>>> print(q.receive()) 6 - Retrieves not-parsed (raw) query result: >>> from binascii import hexlify >>> q.query(qconnection.MessageType.SYNC,'{x}', 10) ->>> print hexlify(q.receive(data_only = True, raw = True)) +>>> print(hexlify(q.receive(data_only = True, raw = True))) fa0a000000 @@ -119,17 +119,18 @@ Both methods accepts the `options` keywords arguments:: >>> query = "{[x] 0Nd, `date$til x}" >>> # retrieve function call as raw byte buffer - >>> print binascii.hexlify(q(query, 5, raw = True)) + >>> from binascii import hexlify + >>> print(binascii.hexlify(q(query, 5, raw = True))) 0e0006000000000000800000000001000000020000000300000004000000 >>> # perform a synchronous call and parse dates vector to numpy array - >>> print q.sync(query, 5, numpy_temporals = True) + >>> print(q.sync(query, 5, numpy_temporals = True)) ['NaT' '2000-01-01' '2000-01-02' '2000-01-03' '2000-01-04' '2000-01-05'] >>> # perform a synchronous call >>> q.query(qconnection.MessageType.SYNC, query, 3) >>> # retrieve query result and represent dates vector as raw data wrapped in QTemporalList - >>> print q.receive(numpy_temporals = False) + >>> print(q.receive(numpy_temporals = False)) [NaT [metadata(qtype=-14)] 2000-01-01 [metadata(qtype=-14)] 2000-01-02 [metadata(qtype=-14)] 2000-01-03 [metadata(qtype=-14)]] diff --git a/doc/source/type-conversion.rst b/doc/source/type-conversion.rst index 6db59b2..fcbd7e4 100644 --- a/doc/source/type-conversion.rst +++ b/doc/source/type-conversion.rst @@ -183,12 +183,12 @@ for ``month``\s etc.) and provides accessors which allow to convert raw data to :: >>> v = q.sync("2001.01.01 2000.05.01 0Nd", numpy_temporals = False) - >>> print '%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v) + >>> print('%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v)) dtype: int32 qtype: -14: [2001-01-01 [metadata(qtype=-14)] 2000-05-01 [metadata(qtype=-14)] NaT [metadata(qtype=-14)]] >>> v = q.sync("2000.01.04D05:36:57.600 0Np", numpy_temporals = False) - >>> print '%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v) + >>> print('%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v)) dtype: int64 qtype: -12: [2000-01-04T05:36:57.600000000+0100 [metadata(qtype=-12)] NaT [metadata(qtype=-12)]] @@ -203,11 +203,11 @@ via :class:`~.qconnection.QConnection` constructor or as parameter to functions: :: >>> v = q.sync("2001.01.01 2000.05.01 0Nd", numpy_temporals = True) - >>> print '%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v) + >>> print('%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v)) dtype: datetime64[D] qtype: -14: ['2001-01-01' '2000-05-01' 'NaT'] >>> v = q.sync("2000.01.04D05:36:57.600 0Np", numpy_temporals = True) - >>> print '%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v) + >>> print('%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v)) dtype: datetime64[ns] qtype: -12: ['2000-01-04T05:36:57.600000000+0100' 'NaT'] diff --git a/doc/source/usage-examples.rst b/doc/source/usage-examples.rst index aa27e16..b82c01e 100644 --- a/doc/source/usage-examples.rst +++ b/doc/source/usage-examples.rst @@ -10,29 +10,30 @@ Following example presents how to execute simple, synchronous query against a re from qpython import qconnection + if __name__ == '__main__': # create connection object - q = qconnection.QConnection(host = 'localhost', port = 5000) + q = qconnection.QConnection(host='localhost', port=5000) # initialize connection q.open() - print q - print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()) + print(q) + print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())) # simple query execution via: QConnection.__call__ data = q('{`int$ til x}', 10) - print 'type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data) - + print('type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data)) + # simple query execution via: QConnection.sync data = q.sync('{`long$ til x}', 10) - print 'type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data) - + print('type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data)) + # low-level query and read - q.query(qconnection.MessageType.SYNC,'{`short$ til x}', 10) # sends a SYNC query - msg = q.receive(data_only = False, raw = False) # retrieve entire message - print 'type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(msg), msg.type, msg.size, msg.is_compressed) + q.query(qconnection.MessageType.SYNC, '{`short$ til x}', 10) # sends a SYNC query + msg = q.receive(data_only=False, raw=False) # retrieve entire message + print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(msg), msg.type, msg.size, msg.is_compressed)) data = msg.data - print 'type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data) + print('type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data)) # close connection q.close() @@ -69,33 +70,33 @@ Following example presents how to execute simple, asynchronous query against a r def __init__(self, q): super(ListenerThread, self).__init__() self.q = q - self._stop = threading.Event() + self._stopper = threading.Event() def stop(self): - self._stop.set() + self._stopper.set() def stopped(self): - return self._stop.isSet() + return self._stopper.isSet() def run(self): while not self.stopped(): - print '.' + print('.') try: message = self.q.receive(data_only = False, raw = False) # retrieve entire message if message.type != MessageType.ASYNC: - print 'Unexpected message, expected message of type: ASYNC' + print('Unexpected message, expected message of type: ASYNC') - print 'type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed) - print message.data + print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed)) + print(message.data) if isinstance(message.data, QDictionary): # stop after 10th query - if message.data['queryid'] == 9: + if message.data[b'queryid'] == 9: self.stop() - except QException, e: - print e + except QException as e: + print(e) if __name__ == '__main__': @@ -104,8 +105,8 @@ Following example presents how to execute simple, asynchronous query against a r # initialize connection q.open() - print q - print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()) + print(q) + print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())) try: # definition of asynchronous multiply function @@ -117,10 +118,10 @@ Following example presents how to execute simple, asynchronous query against a r t = ListenerThread(q) t.start() - for x in xrange(10): + for x in range(10): a = random.randint(1, 100) b = random.randint(1, 100) - print 'Asynchronous call with queryid=%s with arguments: %s, %s' % (x, a, b) + print('Asynchronous call with queryid=%s with arguments: %s, %s' % (x, a, b)) q.async('asynchMult', x, a, b); time.sleep(1) @@ -135,31 +136,38 @@ This example depicts how to create a simple interactive console for communicatio .. code:: python + import qpython from qpython import qconnection from qpython.qtype import QException + try: + input = raw_input + except NameError: + pass + if __name__ == '__main__': + print('qPython %s Cython extensions enabled: %s' % (qpython.__version__, qpython.__is_cython_enabled__)) with qconnection.QConnection(host = 'localhost', port = 5000) as q: - print q - print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()) - + print(q) + print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())) + while True: try: - x = raw_input('Q)') + x = input('Q)') except EOFError: - print + print('') break - + if x == '\\\\': break - + try: result = q(x) - print type(result) - print result - except QException, msg: - print 'q error: \'%s' % msg + print(type(result)) + print(result) + except QException as msg: + print('q error: \'%s' % msg) Twisted integration @@ -182,7 +190,10 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui from qpython.qreader import QReader from qpython.qwriter import QWriter, QWriterException + + class IPCProtocol(Protocol): + class State(object): UNKNOWN = -1 HANDSHAKE = 0 @@ -193,37 +204,37 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui self.credentials = self.factory.username + ':' + self.factory.password if self.factory.password else '' self.transport.write(self.credentials + '\3\0') - + self._message = None def dataReceived(self, data): if self.state == IPCProtocol.State.CONNECTED: try: if not self._message: - self._message = self._reader.read_header(source = data) + self._message = self._reader.read_header(source=data) self._buffer = '' - + self._buffer += data buffer_len = len(self._buffer) if self._buffer else 0 - + while self._message and self._message.size <= buffer_len: complete_message = self._buffer[:self._message.size] - + if buffer_len > self._message.size: self._buffer = self._buffer[self._message.size:] buffer_len = len(self._buffer) if self._buffer else 0 - self._message = self._reader.read_header(source = self._buffer) + self._message = self._reader.read_header(source=self._buffer) else: self._message = None self._buffer = '' buffer_len = 0 - self.factory.onMessage(self._reader.read(source = complete_message)) + self.factory.onMessage(self._reader.read(source=complete_message, numpy_temporals=True)) except: self.factory.onError(sys.exc_info()) self._message = None self._buffer = '' - + elif self.state == IPCProtocol.State.UNKNOWN: # handshake if len(data) == 1: @@ -231,7 +242,7 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui else: self.state = IPCProtocol.State.HANDSHAKE self.transport.write(self.credentials + '\0') - + else: # protocol version fallback if len(data) == 1: @@ -242,8 +253,8 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui def _init(self, data): self.state = IPCProtocol.State.CONNECTED self.protocol_version = min(struct.unpack('B', data)[0], 3) - self._writer = QWriter(stream = None, protocol_version = self.protocol_version) - self._reader = QReader(stream = None) + self._writer = QWriter(stream=None, protocol_version=self.protocol_version) + self._reader = QReader(stream=None) self.factory.clientReady(self) @@ -257,7 +268,9 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui self.transport.write(self._writer.write([query] + list(parameters), msg_type)) + class IPCClientFactory(ClientFactory): + protocol = IPCProtocol def __init__(self, username, password, connect_success_callback, connect_fail_callback, data_callback, error_callback): @@ -271,8 +284,9 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui self.data_callback = data_callback self.error_callback = error_callback + def clientConnectionLost(self, connector, reason): - print 'Lost connection. Reason:', reason + print('Lost connection. Reason: %s' % reason) # connector.connect() def clientConnectionFailed(self, connector, reason): @@ -297,20 +311,25 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui self.client.query(msg_type, query, *parameters) + def onConnectSuccess(source): - print 'Connected, protocol version: ', source.client.protocol_version - source.query(MessageType.SYNC, '.z.ts:{(handle)((1000*(1 ? 100))[0] ? 100)}') + print('Connected, protocol version: %s' % source.client.protocol_version) + source.query(MessageType.SYNC, '.z.ts:{(handle)(`timestamp$100?1000000000000000000)}') source.query(MessageType.SYNC, '.u.sub:{[t;s] handle:: neg .z.w}') source.query(MessageType.ASYNC, '.u.sub', 'trade', '') + def onConnectFail(source, reason): - print 'Connection refused: ', reason + print('Connection refused: %s' % reason) + def onMessage(source, message): - print 'Received: ', message.type, message.data + print('Received: %s %s' % (message.type, message.data)) + def onError(source, error): - print 'Error: ', error + print('Error: %s' % error) + if __name__ == '__main__': factory = IPCClientFactory('user', 'pwd', onConnectSuccess, onConnectFail, onMessage, onError) @@ -325,69 +344,68 @@ This example depicts how to subscribe to standard kdb+ tickerplant service: .. code:: python - import numpy - import threading - import sys - - from qpython import qconnection - from qpython.qtype import QException - from qpython.qconnection import MessageType - from qpython.qcollection import QTable - - - class ListenerThread(threading.Thread): - - def __init__(self, q): - super(ListenerThread, self).__init__() - self.q = q - self._stop = threading.Event() - - def stop(self): - self._stop.set() - - def stopped(self): - return self._stop.isSet() - - def run(self): - while not self.stopped(): - print '.' - try: - message = self.q.receive(data_only = False, raw = False) # retrieve entire message - - if message.type != MessageType.ASYNC: - print 'Unexpected message, expected message of type: ASYNC' - - print 'type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed) - - if isinstance(message.data, list): - # unpack upd message - if len(message.data) == 3 and message.data[0] == 'upd' and isinstance(message.data[2], QTable): - for row in message.data[2]: - print row - - except QException, e: - print e - - - if __name__ == '__main__': - with qconnection.QConnection(host = 'localhost', port = 17010) as q: - print q - print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()) - print 'Press to close application' - - # subscribe to tick - response = q.sync('.u.sub', numpy.string_('trade'), numpy.string_('')) - # get table model - if isinstance(response[1], QTable): - print '%s table data model: %s' % (response[0], response[1].dtype) - - t = ListenerThread(q) - t.start() - - sys.stdin.readline() - - t.stop() - + import numpy + import threading + import sys + + from qpython import qconnection + from qpython.qtype import QException + from qpython.qconnection import MessageType + from qpython.qcollection import QTable + + + class ListenerThread(threading.Thread): + + def __init__(self, q): + super(ListenerThread, self).__init__() + self.q = q + self._stopper = threading.Event() + + def stopit(self): + self._stopper.set() + + def stopped(self): + return self._stopper.is_set() + + def run(self): + while not self.stopped(): + print('.') + try: + message = self.q.receive(data_only = False, raw = False) # retrieve entire message + + if message.type != MessageType.ASYNC: + print('Unexpected message, expected message of type: ASYNC') + + print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed)) + + if isinstance(message.data, list): + # unpack upd message + if len(message.data) == 3 and message.data[0] == b'upd' and isinstance(message.data[2], QTable): + for row in message.data[2]: + print(row) + + except QException as e: + print(e) + + + if __name__ == '__main__': + with qconnection.QConnection(host = 'localhost', port = 17010) as q: + print(q) + print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())) + print('Press to close application') + + # subscribe to tick + response = q.sync('.u.sub', numpy.string_('trade'), numpy.string_('')) + # get table model + if isinstance(response[1], QTable): + print('%s table data model: %s' % (response[0], response[1].dtype)) + + t = ListenerThread(q) + t.start() + + sys.stdin.readline() + + t.stopit() Data publisher @@ -410,59 +428,59 @@ This example shows how to stream data to the kdb+ process using standard tickerp class PublisherThread(threading.Thread): - + def __init__(self, q): super(PublisherThread, self).__init__() self.q = q - self._stop = threading.Event() + self._stopper = threading.Event() def stop(self): - self._stop.set() + self._stopper.set() def stopped(self): - return self._stop.isSet() + return self._stopper.isSet() def run(self): while not self.stopped(): - print '.' + print('.') try: # publish data to tick # function: .u.upd # table: ask self.q.sync('.u.upd', numpy.string_('ask'), self.get_ask_data()) - + time.sleep(1) - except QException, e: - print e + except QException as e: + print(e) except: self.stop() - + def get_ask_data(self): c = random.randint(1, 10) - + today = numpy.datetime64(datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)) - - time = [numpy.timedelta64((numpy.datetime64(datetime.datetime.now()) - today), 'ms') for x in xrange(c)] - instr = ['instr_%d' % random.randint(1, 100) for x in xrange(c)] - src = ['qPython' for x in xrange(c)] - ask = [random.random() * random.randint(1, 100) for x in xrange(c)] - + + time = [numpy.timedelta64((numpy.datetime64(datetime.datetime.now()) - today), 'ms') for x in range(c)] + instr = ['instr_%d' % random.randint(1, 100) for x in range(c)] + src = ['qPython' for x in range(c)] + ask = [random.random() * random.randint(1, 100) for x in range(c)] + data = [qlist(time, qtype=QTIME_LIST), qlist(instr, qtype=QSYMBOL_LIST), qlist(src, qtype=QSYMBOL_LIST), qlist(ask, qtype=QFLOAT_LIST)] - print data + print(data) return data - - + + if __name__ == '__main__': with qconnection.QConnection(host='localhost', port=17010) as q: - print q - print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()) - print 'Press to close application' + print(q) + print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())) + print('Press to close application') t = PublisherThread(q) t.start() - + sys.stdin.readline() - + t.stop() t.join() diff --git a/samples/console.py b/samples/console.py index 595fc57..e44f6ad 100644 --- a/samples/console.py +++ b/samples/console.py @@ -46,4 +46,3 @@ print(result) except QException as msg: print('q error: \'%s' % msg) -