From e64e4b42fa43833ca32029a35fa7041e26b1d0b8 Mon Sep 17 00:00:00 2001 From: Maciej Lach Date: Wed, 17 Jun 2015 15:23:18 +0200 Subject: [PATCH 1/8] exxeleron/qPython#21: Python 2 & 3 compatibility [draft] --- .travis.yml | 1 + CHANGELOG.txt | 6 + README.rst | 2 +- conda.recipe/meta.yaml | 2 +- qpython/__init__.py | 2 +- qpython/_pandas.py | 28 +-- qpython/qcollection.py | 31 ++-- qpython/qconnection.py | 7 +- qpython/qreader.py | 29 +-- qpython/qtemporal.py | 8 +- qpython/qtype.py | 61 ++++--- qpython/qwriter.py | 43 +++-- samples/async_query.py | 40 ++--- samples/console.py | 9 +- samples/publisher.py | 32 ++-- samples/sync_query.py | 16 +- samples/tick_subscriber.py | 42 ++--- samples/twistedclient.py | 10 +- setup.py | 2 +- tests/pandas_test.py | 178 +++++++++---------- tests/qreader_test.py | 356 +++++++++++++++++++------------------ tests/qtypes_test.py | 242 ++++++++++++------------- tests/qwriter_test.py | 224 +++++++++++------------ 23 files changed, 707 insertions(+), 664 deletions(-) diff --git a/.travis.yml b/.travis.yml index bd82ef9..ae7c042 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: python python: - "2.7" + - "3.4" # command to install dependencies install: pip install -r requirements.txt -U diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 835a20a..e461a1f 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,9 @@ +------------------------------------------------------------------------------ + qPython 1.1.0 [2015.xx.xx] +------------------------------------------------------------------------------ + + - Compatibility with Python 2.7 and 3.4 + ------------------------------------------------------------------------------ qPython 1.0.0 [2015.04.10] ------------------------------------------------------------------------------ diff --git a/README.rst b/README.rst index 0f4fd50..0c09ea0 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,7 @@ qPython is a Python library providing support for interprocess communication bet - Support for kdb+ protocol and types: v3.0, v2.6, v<=2.5 - Uncompression of the IPC data stream - Internal representation of data via numpy arrays (lists, complex types) and numpy data types (atoms) -- Supported on Python 2.7 and numpy 1.8 +- Supported on Python 2.7/3.4 and numpy 1.8 For more details please refer to the `documentation`_. diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index e3e0135..f02ef56 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: qpython - version: 1.0.0 + version: 1.1.beta build: number: 1 diff --git a/qpython/__init__.py b/qpython/__init__.py index d017868..6d2a0e0 100644 --- a/qpython/__init__.py +++ b/qpython/__init__.py @@ -48,4 +48,4 @@ def as_dict(self): return self.__dict__.copy() def union_dict(self, **kw): - return dict(self.as_dict().items() + kw.items()) + return dict(list(self.as_dict().items()) + list(kw.items())) diff --git a/qpython/_pandas.py b/qpython/_pandas.py index 4fc960a..bb48b82 100644 --- a/qpython/_pandas.py +++ b/qpython/_pandas.py @@ -16,6 +16,9 @@ import pandas import struct +import sys +if sys.version > '3': + basestring = (str, bytes) from collections import OrderedDict @@ -72,20 +75,25 @@ def _read_table(self, qtype = QTABLE, options = READER_CONFIGURATION): odict = OrderedDict() meta = MetaData(qtype = QTABLE) - for i in xrange(len(columns)): + for i in range(len(columns)): + column_name = columns[i] if isinstance(columns[i], str) else columns[i].decode("utf-8") if isinstance(data[i], str): # convert character list (represented as string) to numpy representation - meta[columns[i]] = QSTRING - odict[columns[i]] = pandas.Series(list(data[i]), dtype = numpy.str).replace(' ', numpy.nan) + meta[column_name] = QSTRING + odict[column_name] = pandas.Series(list(data[i]), dtype = numpy.str).replace(b' ', numpy.nan) + elif isinstance(data[i], bytes): + # convert character list (represented as string) to numpy representation + meta[column_name] = QSTRING + odict[column_name] = pandas.Series(list(data[i].decode()), dtype = numpy.str).replace(b' ', numpy.nan) elif isinstance(data[i], (list, tuple)): - meta[columns[i]] = QGENERAL_LIST + meta[column_name] = QGENERAL_LIST tarray = numpy.ndarray(shape = len(data[i]), dtype = numpy.dtype('O')) - for j in xrange(len(data[i])): + for j in range(len(data[i])): tarray[j] = data[i][j] - odict[columns[i]] = tarray + odict[column_name] = tarray else: - meta[columns[i]] = data[i].meta.qtype - odict[columns[i]] = data[i] + meta[column_name] = data[i].meta.qtype + odict[column_name] = data[i] df = pandas.DataFrame(odict) df.meta = meta @@ -117,7 +125,7 @@ def _read_list(self, qtype, options): def _read_general_list(self, qtype = QGENERAL_LIST, options = READER_CONFIGURATION): list = QReader._read_general_list(self, qtype, options) if options.pandas: - return [numpy.nan if isinstance(element, basestring) and element == ' ' else element for element in list] + return [numpy.nan if isinstance(element, basestring) and element == b' ' else element for element in list] else: return list @@ -147,7 +155,7 @@ def _write_pandas_series(self, data, qtype = None): if qtype is None: # determinate type based on first element of the numpy array qtype = Q_TYPE.get(type(data[0]), QGENERAL_LIST) - + if qtype == QSTRING: # assume we have a generic list of strings -> force representation as symbol list qtype = QSYMBOL diff --git a/qpython/qcollection.py b/qpython/qcollection.py index 0353cab..b9eb38e 100644 --- a/qpython/qcollection.py +++ b/qpython/qcollection.py @@ -14,7 +14,7 @@ # limitations under the License. # -from qtype import * # @UnusedWildImport +from qpython.qtype import * # @UnusedWildImport from qpython import MetaData from qpython.qtemporal import qtemporal, from_raw_qtemporal, to_raw_qtemporal @@ -85,7 +85,7 @@ def get_list_qtype(array): qtype = None - if array.dtype == '|S1': + if str(array.dtype) in ('|S1', 'U1', '|U1') : qtype = QCHAR if qtype is None: @@ -168,7 +168,7 @@ def qlist(array, adjust_dtype = True, **meta): if meta and 'qtype' in meta and meta['qtype'] == QGENERAL_LIST: # force shape and dtype for generic lists tarray = numpy.ndarray(shape = len(array), dtype = numpy.dtype('O')) - for i in xrange(len(array)): + for i in range(len(array)): tarray[i] = array[i] array = tarray else: @@ -268,11 +268,11 @@ def __iter__(self): def items(self): '''Return a copy of the dictionary's list of ``(key, value)`` pairs.''' - return [(self.keys[x], self.values[x]) for x in xrange(len(self.keys))] + return [(self.keys[x], self.values[x]) for x in range(len(self.keys))] def iteritems(self): '''Return an iterator over the dictionary's ``(key, value)`` pairs.''' - for x in xrange(len(self.keys)): + for x in range(len(self.keys)): yield (self.keys[x], self.values[x]) def iterkeys(self): @@ -365,21 +365,26 @@ def qtable(columns, data, **meta): meta['qtype'] = QTABLE dtypes = [] - for i in xrange(len(columns)): + for i in range(len(columns)): + column_name = columns[i] if isinstance(columns[i], str) else columns[i].decode("utf-8") + if isinstance(data[i], str): # convert character list (represented as string) to numpy representation - data[i] = numpy.array(list(data[i]), dtype = numpy.str) + data[i] = numpy.array(list(data[i]), dtype = numpy.string_) + if isinstance(data[i], bytes): + data[i] = numpy.array(list(data[i].decode()), dtype = numpy.string_) - if columns[i] in meta: - data[i] = qlist(data[i], qtype = meta[columns[i]]) + if column_name in meta: + data[i] = qlist(data[i], qtype = meta[column_name]) elif not isinstance(data[i], QList): if type(data[i]) in (list, tuple): data[i] = qlist(data[i], qtype = QGENERAL_LIST) else: data[i] = qlist(data[i]) - meta[columns[i]] = data[i].meta.qtype - dtypes.append((columns[i], data[i].dtype)) + + meta[column_name] = data[i].meta.qtype + dtypes.append((column_name, data[i].dtype)) table = numpy.core.records.fromarrays(data, dtype = dtypes) table = table.view(QTable) @@ -445,11 +450,11 @@ def __iter__(self): def items(self): '''Return a copy of the keyed table's list of ``(key, value)`` pairs.''' - return [(self.keys[x], self.values[x]) for x in xrange(len(self.keys))] + return [(self.keys[x], self.values[x]) for x in range(len(self.keys))] def iteritems(self): '''Return an iterator over the keyed table's ``(key, value)`` pairs.''' - for x in xrange(len(self.keys)): + for x in range(len(self.keys)): yield (self.keys[x], self.values[x]) def iterkeys(self): diff --git a/qpython/qconnection.py b/qpython/qconnection.py index 9890944..b4a85a9 100644 --- a/qpython/qconnection.py +++ b/qpython/qconnection.py @@ -120,7 +120,7 @@ def open(self): self._initialize() self._writer = QWriter(self._connection, protocol_version = self._protocol_version) - self._reader = QReader(self._connection.makefile()) + self._reader = QReader(self._connection.makefile('b')) def _init_socket(self): @@ -157,14 +157,15 @@ def is_connected(self): def _initialize(self): '''Performs a IPC protocol handshake.''' credentials = (self.username if self.username else '') + ':' + (self.password if self.password else '') - self._connection.send(credentials + '\3\0') + credentials = credentials.encode('latin-1') + self._connection.send(credentials + b'\3\0') response = self._connection.recv(1) if len(response) != 1: self.close() self._init_socket() - self._connection.send(credentials + '\0') + self._connection.send(credentials + b'\0') response = self._connection.recv(1) if len(response) != 1: self.close() diff --git a/qpython/qreader.py b/qpython/qreader.py index 2ae7e42..e7aa90a 100644 --- a/qpython/qreader.py +++ b/qpython/qreader.py @@ -16,6 +16,9 @@ import struct import sys +if sys.version > '3': + from sys import intern + unicode = str from qpython import MetaData from qpython.qtype import * # @UnusedWildImport @@ -113,11 +116,11 @@ def __new__(cls, *args, **kwargs): # try to load optional pandas binding try: from qpython._pandas import PandasQReader - return super(QReader, cls).__new__(PandasQReader, args, kwargs) + return super(QReader, cls).__new__(PandasQReader) except ImportError: - return super(QReader, cls).__new__(QReader, args, kwargs) + return super(QReader, cls).__new__(QReader) else: - return super(QReader, cls).__new__(cls, args, kwargs) + return super(QReader, cls).__new__(cls) def __init__(self, stream): @@ -249,17 +252,17 @@ def _read_error(self, qtype = QERROR, options = READER_CONFIGURATION): def _read_string(self, qtype = QSTRING, options = READER_CONFIGURATION): self._buffer.skip() # ignore attributes length = self._buffer.get_int() - return intern(self._buffer.raw(length)) if length > 0 else '' + return self._buffer.raw(length) if length > 0 else b'' @parse(QSYMBOL) def _read_symbol(self, qtype = QSYMBOL, options = READER_CONFIGURATION): - return numpy.string_(intern(self._buffer.get_symbol())) + return numpy.string_(self._buffer.get_symbol()) @parse(QCHAR) def _read_char(self, qtype = QCHAR, options = READER_CONFIGURATION): - return chr(self._read_atom(QCHAR, options)) + return chr(self._read_atom(QCHAR, options)).encode('latin-1') @parse(QGUID) @@ -297,7 +300,7 @@ def _read_list(self, qtype, options): data = numpy.array(symbols, dtype = numpy.string_) return qlist(data, qtype = qtype, adjust_dtype = False) elif qtype == QGUID_LIST: - data = numpy.array([self._read_guid() for x in xrange(length)]) + data = numpy.array([self._read_guid() for x in range(length)]) return qlist(data, qtype = qtype, adjust_dtype = False) elif conversion: raw = self._buffer.raw(length * ATOM_SIZE[qtype]) @@ -340,7 +343,7 @@ def _read_general_list(self, qtype = QGENERAL_LIST, options = READER_CONFIGURATI self._buffer.skip() # ignore attributes length = self._buffer.get_int() - return [self._read_object(options = options) for x in xrange(length)] + return [self._read_object(options = options) for x in range(length)] @parse(QNULL) @@ -356,7 +359,7 @@ def _read_function(self, qtype = QNULL, options = None): def _read_lambda(self, qtype = QLAMBDA, options = READER_CONFIGURATION): self._buffer.get_symbol() # skip expression = self._read_object(options = options) - return QLambda(expression) + return QLambda(expression.decode()) @parse(QCOMPOSITION_FUNC) @@ -388,7 +391,7 @@ def _read_bytes(self, length): raise QReaderException('There is no input data. QReader requires either stream or data chunk') if length == 0: - return '' + return b'' else: data = self._stream.read(length) @@ -511,7 +514,7 @@ def get_symbol(self): :returns: ``\x00`` terminated string ''' - new_position = self._data.find('\x00', self._position) + new_position = self._data.find(b'\x00', self._position) if new_position < 0: raise QReaderException('Failed to read symbol from stream') @@ -537,7 +540,7 @@ def get_symbols(self, count): return [] while c < count: - new_position = self._data.find('\x00', new_position) + new_position = self._data.find(b'\x00', new_position) if new_position < 0: raise QReaderException('Failed to read symbol from stream') @@ -548,6 +551,6 @@ def get_symbols(self, count): raw = self._data[self._position : new_position - 1] self._position = new_position - return raw.split('\x00') + return raw.split(b'\x00') diff --git a/qpython/qtemporal.py b/qpython/qtemporal.py index b088828..bb9042c 100644 --- a/qpython/qtemporal.py +++ b/qpython/qtemporal.py @@ -15,8 +15,8 @@ # from qpython import MetaData -from qtype import * # @UnusedWildImport - +from qpython.qtype import * # @UnusedWildImport +from numpy import longlong _MILLIS_PER_DAY = 24 * 60 * 60 * 1000 _MILLIS_PER_DAY_FLOAT = float(_MILLIS_PER_DAY) @@ -345,7 +345,7 @@ def _to_qtimestamp(dt): if t_dt == numpy.int64: return dt elif t_dt == numpy.datetime64: - return (dt - _EPOCH_TIMESTAMP).astype(long) if not dt == _NUMPY_NULL[QTIMESTAMP] else _QTIMESTAMP_NULL + return (dt - _EPOCH_TIMESTAMP).astype(longlong) if not dt == _NUMPY_NULL[QTIMESTAMP] else _QTIMESTAMP_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -364,7 +364,7 @@ def _to_qtimespan(dt): if t_dt == numpy.int64: return dt elif t_dt == numpy.timedelta64: - return dt.astype(long) if not dt == _NUMPY_NULL[QTIMESPAN] else _QTIMESTAMP_NULL + return dt.astype(longlong) if not dt == _NUMPY_NULL[QTIMESPAN] else _QTIMESTAMP_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) diff --git a/qpython/qtype.py b/qpython/qtype.py index 8ba657a..48b3a72 100644 --- a/qpython/qtype.py +++ b/qpython/qtype.py @@ -1,18 +1,18 @@ -# +# # Copyright (c) 2011-2014 Exxeleron GmbH -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# +# ''' The `qpython.qtype` module defines number of utility function which help to work @@ -81,14 +81,17 @@ QADVERB_FUNC_110 0x6e QADVERB_FUNC_111 0x6f QPROJECTION 0x68 -QERROR -0x80 +QERROR -0x80 ================== ============= ''' import numpy import re import uuid - +import sys +from functools import reduce +if sys.version > '3': + long = int # qtype constants: QNULL = 0x65 @@ -114,7 +117,7 @@ QSTRING_LIST = 0x00 QSYMBOL = -0x0b QSYMBOL_LIST = 0x0b - + QTIMESTAMP = -0x0c QTIMESTAMP_LIST = 0x0c QMONTH = -0x0d @@ -131,7 +134,7 @@ QSECOND_LIST = 0x12 QTIME = -0x13 QTIME_LIST = 0x13 - + QDICTIONARY = 0x63 QKEYED_TABLE = 0x63 QTABLE = 0x62 @@ -148,11 +151,11 @@ QADVERB_FUNC_111 = 0x6f QPROJECTION = 0x68 -QERROR = -0x80 +QERROR = -0x80 -ATOM_SIZE = (0, 1, 16, 0, 1, 2, 4, 8, 4, 8, 1, 0, 8, 4, 4, 8, 8, 4, 4, 4) +ATOM_SIZE = ( 0, 1, 16, 0, 1, 2, 4, 8, 4, 8, 1, 0, 8, 4, 4, 8, 8, 4, 4, 4 ) @@ -251,12 +254,12 @@ # null definitions -_QNULL1 = numpy.int8(-2 ** 7) -_QNULL2 = numpy.int16(-2 ** 15) -_QNULL4 = numpy.int32(-2 ** 31) -_QNULL8 = numpy.int64(-2 ** 63) -_QNAN32 = numpy.fromstring('\x00\x00\xc0\x7f', dtype = numpy.float32)[0] -_QNAN64 = numpy.fromstring('\x00\x00\x00\x00\x00\x00\xf8\x7f', dtype = numpy.float64)[0] +_QNULL1 = numpy.int8(-2**7) +_QNULL2 = numpy.int16(-2**15) +_QNULL4 = numpy.int32(-2**31) +_QNULL8 = numpy.int64(-2**63) +_QNAN32 = numpy.fromstring(b'\x00\x00\xc0\x7f', dtype=numpy.float32)[0] +_QNAN64 = numpy.fromstring(b'\x00\x00\x00\x00\x00\x00\xf8\x7f', dtype=numpy.float64)[0] _QNULL_BOOL = numpy.bool_(False) _QNULL_SYM = numpy.string_('') _QNULL_GUID = uuid.UUID('00000000-0000-0000-0000-000000000000') @@ -271,7 +274,7 @@ QLONG: ('0Nj', _QNULL8, lambda v: v == _QNULL8), QFLOAT: ('0Ne', _QNAN32, lambda v: numpy.isnan(v)), QDOUBLE: ('0n', _QNAN64, lambda v: numpy.isnan(v)), - QSTRING: ('" "', ' ', lambda v: v == ' '), + QSTRING: ('" "', b' ', lambda v: v == b' '), QSYMBOL: ('`', _QNULL_SYM, lambda v: v == _QNULL_SYM), QMONTH: ('0Nm', _QNULL4, lambda v: v == _QNULL4), QDATE: ('0Nd', _QNULL4, lambda v: v == _QNULL4), @@ -287,10 +290,10 @@ def qnull(qtype): '''Retrieve null value for requested q type. - + :Parameters: - `qtype` (`integer`) - qtype indicator - + :returns: null value for specified q type ''' return QNULLMAP[qtype][1] @@ -299,10 +302,10 @@ def qnull(qtype): def is_null(value, qtype): '''Checks whether given value matches null value for a particular q type. - + :Parameters: - `qtype` (`integer`) - qtype indicator - + :returns: `boolean` - ``True`` if value is considered null for given type ``False`` otherwise ''' @@ -331,14 +334,14 @@ def __str__(self): class QLambda(QFunction): '''Represents a q lambda expression. - + .. note:: `expression` is trimmed and required to be valid q function (``{..}``) or k function (``k){..}``). - + :Parameters: - `expression` (`string`) - lambda expression - - :raises: `ValueError` + + :raises: `ValueError` ''' def __init__(self, expression): QFunction.__init__(self, QLAMBDA) @@ -369,7 +372,7 @@ def __eq__(self, other): class QProjection(QFunction): '''Represents a q projection. - + :Parameters: - `parameters` (`list`) - list of parameters for lambda expression ''' @@ -389,7 +392,7 @@ def __str__(self): def __eq__(self, other): return (not self.parameters and not other.parameters) or \ - reduce(lambda v1, v2: v1 or v2, map(lambda v: v in self.parameters, other.parameters)) + reduce(lambda v1,v2: v1 or v2, map(lambda v: v in self.parameters, other.parameters)) def __ne__(self, other): @@ -399,7 +402,7 @@ def __ne__(self, other): class Mapper(object): '''Utility class for creating function execution map via decorators. - + :Parameters: - `call_map` (`dictionary`) - target execution map ''' diff --git a/qpython/qwriter.py b/qpython/qwriter.py index 2ff95fc..3a6cc80 100644 --- a/qpython/qwriter.py +++ b/qpython/qwriter.py @@ -14,12 +14,14 @@ # limitations under the License. # -import cStringIO import struct -import sys +try: + from cStringIO import BytesIO +except ImportError: + from io import BytesIO -from qtype import * # @UnusedWildImport -from qcollection import qlist, QList, QTemporalList, QDictionary, QTable, QKeyedTable, get_list_qtype +from qpython.qtype import * # @UnusedWildImport +from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, QTable, QKeyedTable, get_list_qtype from qpython.qtemporal import QTemporal, to_raw_qtemporal, array_to_raw_qtemporal @@ -52,11 +54,11 @@ def __new__(cls, *args, **kwargs): # try to load optional pandas binding try: from qpython._pandas import PandasQWriter - return super(QWriter, cls).__new__(PandasQWriter, args, kwargs) + return super(QWriter, cls).__new__(PandasQWriter) except ImportError: - return super(QWriter, cls).__new__(QWriter, args, kwargs) + return super(QWriter, cls).__new__(QWriter) else: - return super(QWriter, cls).__new__(cls, args, kwargs) + return super(QWriter, cls).__new__(cls) def __init__(self, stream, protocol_version): @@ -75,10 +77,10 @@ def write(self, data, msg_type): :returns: if wraped stream is ``None`` serialized data, otherwise ``None`` ''' - self._buffer = cStringIO.StringIO() + self._buffer = BytesIO() # header and placeholder for message size - self._buffer.write('%s%s\0\0\0\0\0\0' % (ENDIANESS, chr(msg_type))) + self._buffer.write(('%s%s\0\0\0\0\0\0' % (ENDIANESS, chr(msg_type))).encode("latin-1")) self._write(data) @@ -125,13 +127,13 @@ def _write_error(self, data): self._buffer.write(struct.pack('b', QERROR)) if isinstance(data, Exception): msg = data.__class__.__name__ - if data.message: - msg = data.message + if data.args: + msg = data.args[0] else: msg = data.__name__ - self._buffer.write(msg) - self._buffer.write('\0') + self._buffer.write(msg.encode("latin-1")) + self._buffer.write(b'\0') def _write_atom(self, data, qtype): @@ -150,13 +152,16 @@ def _write_generic_list(self, data): self._write(element) - @serialize(str) + @serialize(str, bytes) def _write_string(self, data): if len(data) == 1: self._write_atom(ord(data), QCHAR) else: self._buffer.write(struct.pack('=bxi', QSTRING, len(data))) - self._buffer.write(data) + if isinstance(data, str): + self._buffer.write(data.encode("latin-1")) + else: + self._buffer.write(data) @serialize(numpy.string_) @@ -164,7 +169,7 @@ def _write_symbol(self, data): self._buffer.write(struct.pack('=b', QSYMBOL)) if data: self._buffer.write(data) - self._buffer.write('\0') + self._buffer.write(b'\0') @serialize(uuid.UUID) @@ -207,7 +212,7 @@ def _write_numpy_temporal(self, data): @serialize(QLambda) def _write_lambda(self, data): self._buffer.write(struct.pack('=b', QLAMBDA)) - self._buffer.write('\0') + self._buffer.write(b'\0') self._write_string(data.expression) @@ -257,7 +262,9 @@ def _write_list(self, data, qtype = None): if qtype == QSYMBOL: for symbol in data: - self._buffer.write('%s\0' % (symbol or '')) + if symbol: + self._buffer.write(symbol) + self._buffer.write(b'\0') elif qtype == QGUID: if self._protocol_version < 3: raise QWriterException('kdb+ protocol version violation: Guid not supported pre kdb+ v3.0') diff --git a/samples/async_query.py b/samples/async_query.py index be2b7d2..1f58c8a 100644 --- a/samples/async_query.py +++ b/samples/async_query.py @@ -1,18 +1,18 @@ -# +# # Copyright (c) 2011-2014 Exxeleron GmbH -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# +# import random import threading @@ -25,36 +25,36 @@ class ListenerThread(threading.Thread): - + def __init__(self, q): super(ListenerThread, self).__init__() self.q = q - self._stop = threading.Event() + self._stopper = threading.Event() def stop(self): - self._stop.set() + self._stopper.set() def stopped(self): - return self._stop.isSet() + return self._stopper.isSet() def run(self): while not self.stopped(): print('.') try: - message = self.q.receive(data_only = False, raw = False) # retrieve entire message - + message = self.q.receive(data_only = False, raw = False) # retrieve entire message + if message.type != MessageType.ASYNC: print('Unexpected message, expected message of type: ASYNC') - + print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed)) print(message.data) - + if isinstance(message.data, QDictionary): # stop after 10th query - if message.data['queryid'] == 9: + if message.data[b'queryid'] == 9: self.stop() - - except QException, e: + + except QException as e: print(e) @@ -76,13 +76,13 @@ def run(self): t = ListenerThread(q) t.start() - - for x in xrange(10): + + for x in range(10): a = random.randint(1, 100) b = random.randint(1, 100) print('Asynchronous call with queryid=%s with arguments: %s, %s' % (x, a, b)) - q.async('asynchMult', x, a, b) - + q.async('asynchMult', x, a, b); + time.sleep(1) finally: q.close() diff --git a/samples/console.py b/samples/console.py index 33cc264..0797827 100644 --- a/samples/console.py +++ b/samples/console.py @@ -17,6 +17,11 @@ from qpython import qconnection from qpython.qtype import QException +try: + input = raw_input +except NameError: + pass + if __name__ == '__main__': with qconnection.QConnection(host = 'localhost', port = 5000) as q: @@ -25,7 +30,7 @@ while True: try: - x = raw_input('Q)') + x = input('Q)') except EOFError: print('') break @@ -37,6 +42,6 @@ result = q(x) print(type(result)) print(result) - except QException, msg: + except QException as msg: print('q error: \'%s' % msg) diff --git a/samples/publisher.py b/samples/publisher.py index d42fadb..4ebbf70 100644 --- a/samples/publisher.py +++ b/samples/publisher.py @@ -1,18 +1,18 @@ -# +# # Copyright (c) 2011-2014 Exxeleron GmbH -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# +# import datetime import numpy @@ -31,13 +31,13 @@ class PublisherThread(threading.Thread): def __init__(self, q): super(PublisherThread, self).__init__() self.q = q - self._stop = threading.Event() + self._stopper = threading.Event() def stop(self): - self._stop.set() + self._stopper.set() def stopped(self): - return self._stop.isSet() + return self._stopper.isSet() def run(self): while not self.stopped(): @@ -49,7 +49,7 @@ def run(self): self.q.sync('.u.upd', numpy.string_('ask'), self.get_ask_data()) time.sleep(1) - except QException, e: + except QException as e: print(e) except: self.stop() @@ -57,20 +57,20 @@ def run(self): def get_ask_data(self): c = random.randint(1, 10) - today = numpy.datetime64(datetime.datetime.now().replace(hour = 0, minute = 0, second = 0, microsecond = 0)) + today = numpy.datetime64(datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)) - time = [numpy.timedelta64((numpy.datetime64(datetime.datetime.now()) - today), 'ms') for x in xrange(c)] - instr = ['instr_%d' % random.randint(1, 100) for x in xrange(c)] - src = ['qPython' for x in xrange(c)] - ask = [random.random() * random.randint(1, 100) for x in xrange(c)] + time = [numpy.timedelta64((numpy.datetime64(datetime.datetime.now()) - today), 'ms') for x in range(c)] + instr = ['instr_%d' % random.randint(1, 100) for x in range(c)] + src = ['qPython' for x in range(c)] + ask = [random.random() * random.randint(1, 100) for x in range(c)] - data = [qlist(time, qtype = QTIME_LIST), qlist(instr, qtype = QSYMBOL_LIST), qlist(src, qtype = QSYMBOL_LIST), qlist(ask, qtype = QFLOAT_LIST)] + data = [qlist(time, qtype=QTIME_LIST), qlist(instr, qtype=QSYMBOL_LIST), qlist(src, qtype=QSYMBOL_LIST), qlist(ask, qtype=QFLOAT_LIST)] print(data) return data if __name__ == '__main__': - with qconnection.QConnection(host = 'localhost', port = 17010) as q: + with qconnection.QConnection(host='localhost', port=17010) as q: print(q) print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())) print('Press to close application') diff --git a/samples/sync_query.py b/samples/sync_query.py index 86d53b4..8710668 100644 --- a/samples/sync_query.py +++ b/samples/sync_query.py @@ -1,25 +1,25 @@ -# +# # Copyright (c) 2011-2014 Exxeleron GmbH -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# +# from qpython import qconnection if __name__ == '__main__': # create connection object - q = qconnection.QConnection(host = 'localhost', port = 5000) + q = qconnection.QConnection(host='localhost', port=5000) # initialize connection q.open() @@ -35,8 +35,8 @@ print('type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data)) # low-level query and read - q.query(qconnection.MessageType.SYNC, '{`short$ til x}', 10) # sends a SYNC query - msg = q.receive(data_only = False, raw = False) # retrieve entire message + q.query(qconnection.MessageType.SYNC, '{`short$ til x}', 10) # sends a SYNC query + msg = q.receive(data_only=False, raw=False) # retrieve entire message print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(msg), msg.type, msg.size, msg.is_compressed)) data = msg.data print('type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data)) diff --git a/samples/tick_subscriber.py b/samples/tick_subscriber.py index f48362e..279f387 100644 --- a/samples/tick_subscriber.py +++ b/samples/tick_subscriber.py @@ -1,18 +1,18 @@ -# +# # Copyright (c) 2011-2014 Exxeleron GmbH -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# +# import numpy import threading @@ -25,36 +25,36 @@ class ListenerThread(threading.Thread): - + def __init__(self, q): super(ListenerThread, self).__init__() self.q = q - self._stop = threading.Event() + self._stopper = threading.Event() - def stop(self): - self._stop.set() + def stopit(self): + self._stopper.set() def stopped(self): - return self._stop.isSet() + return self._stopper.is_set() def run(self): while not self.stopped(): print('.') try: - message = self.q.receive(data_only = False, raw = False) # retrieve entire message - + message = self.q.receive(data_only = False, raw = False) # retrieve entire message + if message.type != MessageType.ASYNC: print('Unexpected message, expected message of type: ASYNC') - + print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed)) - + if isinstance(message.data, list): # unpack upd message - if len(message.data) == 3 and message.data[0] == 'upd' and isinstance(message.data[2], QTable): + if len(message.data) == 3 and message.data[0] == b'upd' and isinstance(message.data[2], QTable): for row in message.data[2]: print(row) - - except QException, e: + + except QException as e: print(e) @@ -66,13 +66,13 @@ def run(self): # subscribe to tick response = q.sync('.u.sub', numpy.string_('trade'), numpy.string_('')) - # get table model + # get table model if isinstance(response[1], QTable): print('%s table data model: %s' % (response[0], response[1].dtype)) t = ListenerThread(q) t.start() - + sys.stdin.readline() - - t.stop() + + t.stopit() diff --git a/samples/twistedclient.py b/samples/twistedclient.py index 02eb63c..16f0316 100644 --- a/samples/twistedclient.py +++ b/samples/twistedclient.py @@ -45,7 +45,7 @@ def dataReceived(self, data): if self.state == IPCProtocol.State.CONNECTED: try: if not self._message: - self._message = self._reader.read_header(source = data) + self._message = self._reader.read_header(source=data) self._buffer = '' self._buffer += data @@ -57,13 +57,13 @@ def dataReceived(self, data): if buffer_len > self._message.size: self._buffer = self._buffer[self._message.size:] buffer_len = len(self._buffer) if self._buffer else 0 - self._message = self._reader.read_header(source = self._buffer) + self._message = self._reader.read_header(source=self._buffer) else: self._message = None self._buffer = '' buffer_len = 0 - self.factory.onMessage(self._reader.read(source = complete_message, numpy_temporals = True)) + self.factory.onMessage(self._reader.read(source=complete_message, numpy_temporals=True)) except: self.factory.onError(sys.exc_info()) self._message = None @@ -87,8 +87,8 @@ def dataReceived(self, data): def _init(self, data): self.state = IPCProtocol.State.CONNECTED self.protocol_version = min(struct.unpack('B', data)[0], 3) - self._writer = QWriter(stream = None, protocol_version = self.protocol_version) - self._reader = QReader(stream = None) + self._writer = QWriter(stream=None, protocol_version=self.protocol_version) + self._reader = QReader(stream=None) self.factory.clientReady(self) diff --git a/setup.py b/setup.py index f75f59f..d65e017 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ def read(fname): setup(name = 'qPython', - version = '1.0.0', + version = '1.1 beta', description = 'kdb+ interfacing library for Python', long_description=read('README.rst'), diff --git a/tests/pandas_test.py b/tests/pandas_test.py index 8be85c3..e5daed6 100644 --- a/tests/pandas_test.py +++ b/tests/pandas_test.py @@ -15,9 +15,12 @@ # import binascii -import cStringIO import struct import sys +try: + from cStringIO import BytesIO +except ImportError: + from io import BytesIO from collections import OrderedDict from qpython import qreader, MetaData, qwriter @@ -32,126 +35,126 @@ BINARY = None PANDAS_EXPRESSIONS = OrderedDict(( - ('("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)', + (b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)', {'data': pandas.Series(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), numpy.nan])), 'meta': MetaData(qtype = QGUID_LIST) }), - ('"quick brown fox jumps over a lazy dog"', 'quick brown fox jumps over a lazy dog'), - ('" "', ' '), - ('``quick``fox', {'data': pandas.Series(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')])), + (b'"quick brown fox jumps over a lazy dog"', b'quick brown fox jumps over a lazy dog'), + (b'" "', b' '), + (b'``quick``fox', {'data': pandas.Series(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')])), 'meta': MetaData(qtype = QSYMBOL_LIST) }), - ('`the`quick`brown`fox', {'data': pandas.Series(numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')])), + (b'`the`quick`brown`fox', {'data': pandas.Series(numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')])), 'meta': MetaData(qtype = QSYMBOL_LIST) }), - ('("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")', - ['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog']), - ('("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")', - ['quick', numpy.nan, 'fox', 'jumps', 'over', 'a lazy', 'dog']), + (b'("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")', + [b'quick', b'brown', b'fox', b'jumps', b'over', b'a lazy', b'dog']), + (b'("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")', + [b'quick', numpy.nan, b'fox', b'jumps', b'over', b'a lazy', b'dog']), - ('(0b;1b;0b)', {'data': pandas.Series(numpy.array([False, True, False], dtype = numpy.bool)), + (b'(0b;1b;0b)', {'data': pandas.Series(numpy.array([False, True, False], dtype = numpy.bool)), 'meta': MetaData(qtype = QBOOL_LIST) }), - ('(0x01;0x02;0xff)', {'data': pandas.Series(numpy.array([1, 2, 0xff], dtype = numpy.int8)), + (b'(0x01;0x02;0xff)', {'data': pandas.Series(numpy.array([1, 2, 0xff], dtype = numpy.int8)), 'meta': MetaData(qtype = QBYTE_LIST) }), - ('(1h;2h;3h)', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int16)), + (b'(1h;2h;3h)', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int16)), 'meta': MetaData(qtype = QSHORT_LIST) }), - ('(1h;0Nh;3h)', {'data': pandas.Series(numpy.array([1, numpy.nan, 3])), + (b'(1h;0Nh;3h)', {'data': pandas.Series(numpy.array([1, numpy.nan, 3])), 'meta': MetaData(qtype = QSHORT_LIST) }), - ('1 2 3', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64)), + (b'1 2 3', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64)), 'meta': MetaData(qtype = QLONG_LIST) }), - ('1 0N 3', {'data': pandas.Series([1, numpy.nan, 3]), + (b'1 0N 3', {'data': pandas.Series([1, numpy.nan, 3]), 'meta': MetaData(qtype = QLONG_LIST) }), - ('(1i;2i;3i)', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int32)), + (b'(1i;2i;3i)', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int32)), 'meta': MetaData(qtype = QINT_LIST) }), - ('(1i;0Ni;3i)', {'data': pandas.Series(numpy.array([1, numpy.nan, 3])), + (b'(1i;0Ni;3i)', {'data': pandas.Series(numpy.array([1, numpy.nan, 3])), 'meta': MetaData(qtype = QINT_LIST) }), - ('(1j;2j;3j)', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64)), + (b'(1j;2j;3j)', {'data': pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64)), 'meta': MetaData(qtype = QLONG_LIST) }), - ('(1j;0Nj;3j)', {'data': pandas.Series(numpy.array([1, numpy.nan, 3])), + (b'(1j;0Nj;3j)', {'data': pandas.Series(numpy.array([1, numpy.nan, 3])), 'meta': MetaData(qtype = QLONG_LIST) }), - ('(5.5e; 8.5e)', {'data': pandas.Series(numpy.array([5.5, 8.5]), dtype = numpy.float32), + (b'(5.5e; 8.5e)', {'data': pandas.Series(numpy.array([5.5, 8.5]), dtype = numpy.float32), 'meta': MetaData(qtype = QFLOAT_LIST) }), - ('(5.5e; 0Ne)', {'data': pandas.Series(numpy.array([5.5, numpy.nan]), dtype = numpy.float32), + (b'(5.5e; 0Ne)', {'data': pandas.Series(numpy.array([5.5, numpy.nan]), dtype = numpy.float32), 'meta': MetaData(qtype = QFLOAT_LIST) }), - ('3.23 6.46', {'data': pandas.Series(numpy.array([3.23, 6.46])), + (b'3.23 6.46', {'data': pandas.Series(numpy.array([3.23, 6.46])), 'meta': MetaData(qtype = QDOUBLE_LIST) }), - ('3.23 0n', {'data': pandas.Series(numpy.array([3.23, numpy.nan])), + (b'3.23 0n', {'data': pandas.Series(numpy.array([3.23, numpy.nan])), 'meta': MetaData(qtype = QDOUBLE_LIST) }), - ('(2001.01m; 0Nm)', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]')), + (b'(2001.01m; 0Nm)', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]')), 'meta': MetaData(qtype = QMONTH_LIST) }), - ('2001.01.01 2000.05.01 0Nd', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')), + (b'2001.01.01 2000.05.01 0Nd', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')), 'meta': MetaData(qtype = QDATE_LIST) }), - ('2000.01.04T05:36:57.600 0Nz', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')])), + (b'2000.01.04T05:36:57.600 0Nz', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')])), 'meta': MetaData(qtype = QDATETIME_LIST) }), - ('12:01 0Nu', {'data': pandas.Series(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')])), + (b'12:01 0Nu', {'data': pandas.Series(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')])), 'meta': MetaData(qtype = QMINUTE_LIST) }), - ('12:05:00 0Nv', {'data': pandas.Series(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')])), + (b'12:05:00 0Nv', {'data': pandas.Series(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')])), 'meta': MetaData(qtype = QSECOND_LIST) }), - ('12:04:59.123 0Nt', {'data': pandas.Series(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')])), + (b'12:04:59.123 0Nt', {'data': pandas.Series(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')])), 'meta': MetaData(qtype = QTIME_LIST) }), - ('2000.01.04D05:36:57.600 0Np', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')])), + (b'2000.01.04D05:36:57.600 0Np', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')])), 'meta': MetaData(qtype = QTIMESTAMP_LIST) }), - ('0D05:36:57.600 0Nn', {'data': pandas.Series(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')])), + (b'0D05:36:57.600 0Nn', {'data': pandas.Series(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')])), 'meta': MetaData(qtype = QTIMESPAN_LIST) }), - ('1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST), + (b'1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))), - ('(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)], + (b'(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)], qlist(numpy.array(['first', 'second']), qtype = QSYMBOL_LIST))), - ('(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4'], - [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])), - ('`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST), + (b'(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), b'4'], + [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), b'456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])), + (b'`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST), [[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), [numpy.string_('a'), numpy.int64(2)]), numpy.float32(5.5)])), - ('flip `abc`def!(1 2 3; 4 5 6)', {'data': pandas.DataFrame(OrderedDict((('abc', pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64))), + (b'flip `abc`def!(1 2 3; 4 5 6)', {'data': pandas.DataFrame(OrderedDict((('abc', pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64))), ('def', pandas.Series(numpy.array([4, 5, 6], dtype = numpy.int64))))) ), 'meta': MetaData(**{'qtype': QTABLE, 'abc': QLONG_LIST, 'def': QLONG_LIST}) }), - ('flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)', - {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'])), + (b'flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)', + {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'], dtype = numpy.string_)), ('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64))))) ), 'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QLONG_LIST}) }), - ('flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")', - {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'])), + (b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")', + {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'], dtype = numpy.string_)), ('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64))), - ('grade', pandas.Series(["a", numpy.nan,"c"])), + ('grade', pandas.Series(['a', ' ', 'c'], dtype = numpy.str).replace(b' ', numpy.nan)), )) ), 'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QLONG_LIST, 'grade': QSTRING}) }), - ('flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))', - {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'])), + (b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))', + {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'], dtype = numpy.string_)), ('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64))), - ('fullname', pandas.Series(["Arthur Dent", "Zaphod Beeblebrox", "Ford Prefect"])), + ('fullname', pandas.Series(["Arthur Dent", "Zaphod Beeblebrox", "Ford Prefect"], dtype = numpy.string_)), )) ), 'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QLONG_LIST, 'fullname': QSTRING_LIST}) }), - ('flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; " "; "Ford Prefect"))', - {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'])), + (b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; " "; "Ford Prefect"))', + {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'], dtype = numpy.string_)), ('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64))), - ('fullname', pandas.Series(["Arthur Dent", numpy.nan, "Ford Prefect"])), + ('fullname', pandas.Series([b"Arthur Dent", numpy.nan, b"Ford Prefect"])), )) ), 'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QLONG_LIST, 'fullname': QSTRING_LIST}) }), - ('([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', {'data': pandas.DataFrame(OrderedDict((('sc', pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64))), + (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', {'data': pandas.DataFrame(OrderedDict((('sc', pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64))), ('nsc', [pandas.Series(numpy.array([1, 2], dtype = numpy.int64)), pandas.Series(numpy.array([3, 4], dtype = numpy.int64)), pandas.Series(numpy.array([5, 6, 7], dtype = numpy.int64))]))) ), 'meta': MetaData(**{'qtype': QTABLE, 'nsc': QGENERAL_LIST, 'sc': QLONG_LIST}) }), - ('([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', {'data': pandas.DataFrame(OrderedDict((('sc', pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64))), + (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', {'data': pandas.DataFrame(OrderedDict((('sc', pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64))), ('nsc', [pandas.Series(numpy.array([1, 2], dtype = numpy.int64)), pandas.Series(numpy.array([3, 4], dtype = numpy.int64)), pandas.Series(numpy.array([5, 6], dtype = numpy.int64))]))) ), 'meta': MetaData(**{'qtype': QTABLE, 'nsc': QGENERAL_LIST, 'sc': QLONG_LIST}) }), - ('([] name:`symbol$(); iq:`int$())', {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(numpy.array([], dtype = numpy.string_))), + (b'([] name:`symbol$(); iq:`int$())', {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(numpy.array([], dtype = numpy.string_))), ('iq', pandas.Series(numpy.array([], dtype = numpy.int32))))) ), 'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QINT_LIST}) }), - ('([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', - {'data': pandas.DataFrame(OrderedDict((('pos', pandas.Series(numpy.array(['d1', 'd2', 'd3']))), + (b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', + {'data': pandas.DataFrame(OrderedDict((('pos', pandas.Series(numpy.array(['d1', 'd2', 'd3'], dtype = numpy.string_))), ('dates', pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'))))) ), 'meta': MetaData(**{'qtype': QTABLE, 'pos': QSYMBOL_LIST, 'dates': QDATE_LIST}) }), - ('([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', + (b'([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', {'data': pandas.DataFrame(OrderedDict((('eid', pandas.Series(numpy.array([1001, 1002, 1003], dtype = numpy.int64))), - ('pos', pandas.Series(numpy.array(['d1', 'd2', 'd3']))), + ('pos', pandas.Series(numpy.array(['d1', 'd2', 'd3'], dtype = numpy.string_))), ('dates', pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'))))) ), 'meta': MetaData(**{'qtype': QKEYED_TABLE, 'pos': QSYMBOL_LIST, 'dates': QDATE_LIST, 'eid': QLONG_LIST}), @@ -160,27 +163,16 @@ PANDAS_EXPRESSIONS_ALT = OrderedDict(( - ('("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")', + (b'("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")', {'data': pandas.Series(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog']), 'meta': MetaData(qtype = QSTRING_LIST) }), - ('`the`quick`brown`fox', ({'data': pandas.Series(numpy.array(['the', 'quick', 'brown', 'fox'])), - 'meta': MetaData(qtype = QSYMBOL_LIST) }, - {'data': pandas.Series(['the', 'quick', 'brown', 'fox']), - 'meta': MetaData(qtype = QSYMBOL_LIST) }, - pandas.Series(['the', 'quick', 'brown', 'fox']) - )), - ('flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)', - pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'])), - ('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64))), - ))), - ), )) def arrays_equal(left, right): if type(left) != type(right): return False - if type(left) in [numpy.ndarray, pandas.Series] and left.dtype != right.dtype: + if type(left) in [numpy.ndarray, pandas.Series] and left.dtype.type != right.dtype.type: print('Type comparison failed: %s != %s' % (left.dtype, right.dtype)) return False @@ -191,7 +183,7 @@ def arrays_equal(left, right): if len(left) != len(right): return False - for i in xrange(len(left)): + for i in range(len(left)): if type(left[i]) != type(right[i]): print('Type comparison failed: %s != %s' % (type(left[i]), type(right[i]))) return False @@ -241,11 +233,11 @@ def init(): def test_reading_pandas(): print('Deserialization (pandas)') - for query, value in PANDAS_EXPRESSIONS.iteritems(): - buffer_ = cStringIO.StringIO() + for query, value in iter(PANDAS_EXPRESSIONS.items()): + buffer_ = BytesIO() binary = binascii.unhexlify(BINARY[query]) - buffer_.write('\1\0\0\0') + buffer_.write(b'\1\0\0\0') buffer_.write(struct.pack('i', len(binary) + 8)) buffer_.write(binary) buffer_.seek(0) @@ -266,7 +258,7 @@ def test_reading_pandas(): else: assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result) print('.') - except QException, e: + except QException as e: assert isinstance(value, QException) assert e.message == value.message print('.') @@ -275,8 +267,8 @@ def test_reading_pandas(): def test_writing_pandas(): w = qwriter.QWriter(None, 3) - for query, value in PANDAS_EXPRESSIONS.iteritems(): - sys.stdout.write('%-75s' % query) + for query, value in iter(PANDAS_EXPRESSIONS.items()): + sys.stdout.write( '%-75s' % query ) if isinstance(value, dict): data = value['data'] if 'index' in value: @@ -286,28 +278,24 @@ def test_writing_pandas(): else: data = value serialized = binascii.hexlify(w.write(data, 1))[16:].lower() - assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (value, BINARY[query].lower(), serialized) - sys.stdout.write('.') + assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (value, BINARY[query].lower(), serialized) + sys.stdout.write( '.' ) print('') - for query, variants in PANDAS_EXPRESSIONS_ALT.iteritems(): - sys.stdout.write('%-75s' % query) - variants = [variants] if not isinstance(variants, tuple) else variants - - for value in variants: - if isinstance(value, dict): - data = value['data'] - if 'index' in value: - data.reset_index(drop = True) - data = data.set_index(value['index']) - data.meta = value['meta'] - else: - data = value - serialized = binascii.hexlify(w.write(data, 1))[16:].lower() - assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (value, BINARY[query].lower(), serialized) - - sys.stdout.write('.') + for query, value in iter(PANDAS_EXPRESSIONS_ALT.items()): + sys.stdout.write( '%-75s' % query ) + if isinstance(value, dict): + data = value['data'] + if 'index' in value: + data.reset_index(drop = True) + data = data.set_index(value['index']) + data.meta = value['meta'] + else: + data = value + serialized = binascii.hexlify(w.write(data, 1))[16:].lower() + assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (value, BINARY[query].lower(), serialized) + sys.stdout.write( '.' ) print('') diff --git a/tests/qreader_test.py b/tests/qreader_test.py index d2e222f..432c92c 100644 --- a/tests/qreader_test.py +++ b/tests/qreader_test.py @@ -15,10 +15,16 @@ # import binascii -import cStringIO import struct import sys - +try: + from cStringIO import BytesIO +except ImportError: + from io import BytesIO + +if sys.version > '3': + long = int + from collections import OrderedDict from qpython import qreader from qpython.qtype import * # @UnusedWildImport @@ -28,152 +34,152 @@ EXPRESSIONS = OrderedDict(( - ('("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)', + (b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)', qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST)), - ('"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')), - ('"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')), - ('(2001.01m; 0Nm)', qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST)), - ('2001.01m', qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH)), - ('0Nm', qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH)), - ('2001.01.01 2000.05.01 0Nd', qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)), - ('2001.01.01', qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE)), - ('0Nd', qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE)), - ('2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST)), - ('2000.01.04T05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), qtype=QDATETIME)), - ('0Nz', qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME)), - ('12:01 0Nu', qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST)), - ('12:01', qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE)), - ('0Nu', qtemporal(numpy.timedelta64('NaT', 'm'), qtype=QMINUTE)), - ('12:05:00 0Nv', qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST)), - ('12:05:00', qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND)), - ('0Nv', qtemporal(numpy.timedelta64('NaT', 's'), qtype=QSECOND)), - ('12:04:59.123 0Nt', qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST)), - ('12:04:59.123', qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME)), - ('0Nt', qtemporal(numpy.timedelta64('NaT', 'ms'), qtype=QTIME)), - ('2000.01.04D05:36:57.600 0Np', qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST)), - ('2000.01.04D05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), qtype=QTIMESTAMP)), - ('0Np', qtemporal(numpy.datetime64('NaT', 'ns'), qtype=QTIMESTAMP)), - ('0D05:36:57.600 0Nn', qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST)), - ('0D05:36:57.600', qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN)), - ('0Nn', qtemporal(numpy.timedelta64('NaT', 'ns'), qtype=QTIMESPAN)), + (b'"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')), + (b'"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')), + (b'(2001.01m; 0Nm)', qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST)), + (b'2001.01m', qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH)), + (b'0Nm', qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH)), + (b'2001.01.01 2000.05.01 0Nd', qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)), + (b'2001.01.01', qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE)), + (b'0Nd', qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE)), + (b'2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST)), + (b'2000.01.04T05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), qtype=QDATETIME)), + (b'0Nz', qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME)), + (b'12:01 0Nu', qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST)), + (b'12:01', qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE)), + (b'0Nu', qtemporal(numpy.timedelta64('NaT', 'm'), qtype=QMINUTE)), + (b'12:05:00 0Nv', qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST)), + (b'12:05:00', qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND)), + (b'0Nv', qtemporal(numpy.timedelta64('NaT', 's'), qtype=QSECOND)), + (b'12:04:59.123 0Nt', qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST)), + (b'12:04:59.123', qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME)), + (b'0Nt', qtemporal(numpy.timedelta64('NaT', 'ms'), qtype=QTIME)), + (b'2000.01.04D05:36:57.600 0Np', qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST)), + (b'2000.01.04D05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), qtype=QTIMESTAMP)), + (b'0Np', qtemporal(numpy.datetime64('NaT', 'ns'), qtype=QTIMESTAMP)), + (b'0D05:36:57.600 0Nn', qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST)), + (b'0D05:36:57.600', qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN)), + (b'0Nn', qtemporal(numpy.timedelta64('NaT', 'ns'), qtype=QTIMESPAN)), - ('::', None), - ('1+`', QException('type')), - ('1', numpy.int64(1)), - ('1i', numpy.int32(1)), - ('-234h', numpy.int16(-234)), - ('0b', numpy.bool_(False)), - ('1b', numpy.bool_(True)), - ('0x2a', numpy.byte(0x2a)), - ('89421099511627575j', numpy.int64(89421099511627575L)), - ('5.5e', numpy.float32(5.5)), - ('3.234', numpy.float64(3.234)), - ('"0"', '0'), - ('"abc"', 'abc'), - ('"quick brown fox jumps over a lazy dog"', 'quick brown fox jumps over a lazy dog'), - ('`abc', numpy.string_('abc')), - ('`quickbrownfoxjumpsoveralazydog', numpy.string_('quickbrownfoxjumpsoveralazydog')), - ('0Nh', qnull(QSHORT)), - ('0N', qnull(QLONG)), - ('0Ni', qnull(QINT)), - ('0Nj', qnull(QLONG)), - ('0Ne', qnull(QFLOAT)), - ('0n', qnull(QDOUBLE)), - ('" "', qnull(QSTRING)), - ('`', qnull(QSYMBOL)), - ('0Ng', qnull(QGUID)), - ('()', []), - ('(0b;1b;0b)', qlist(numpy.array([False, True, False], dtype=numpy.bool_), qtype=QBOOL_LIST)), - ('(0x01;0x02;0xff)', qlist(numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), qtype=QBYTE_LIST)), - ('(1h;2h;3h)', qlist(numpy.array([1, 2, 3], dtype=numpy.int16), qtype=QSHORT_LIST)), - ('(1h;0Nh;3h)', qlist(numpy.array([1, qnull(QSHORT), 3], dtype=numpy.int16), qtype=QSHORT_LIST)), - ('1 2 3', qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST)), - ('1 0N 3', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)), - ('(1i;2i;3i)', qlist(numpy.array([1, 2, 3], dtype=numpy.int32), qtype=QINT_LIST)), - ('(1i;0Ni;3i)', qlist(numpy.array([1, qnull(QINT), 3], dtype=numpy.int32), qtype=QINT_LIST)), - ('(1j;2j;3j)', qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST)), - ('(1j;0Nj;3j)', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)), - ('(5.5e; 8.5e)', qlist(numpy.array([5.5, 8.5], dtype=numpy.float32), qtype=QFLOAT_LIST)), - ('(5.5e; 0Ne)', qlist(numpy.array([5.5, qnull(QFLOAT)], dtype=numpy.float32), qtype=QFLOAT_LIST)), - ('3.23 6.46', qlist(numpy.array([3.23, 6.46], dtype=numpy.float64), qtype=QDOUBLE_LIST)), - ('3.23 0n', qlist(numpy.array([3.23, qnull(QDOUBLE)], dtype=numpy.float64), qtype=QDOUBLE_LIST)), - ('(1;`bcd;"0bc";5.5e)', [numpy.int64(1), numpy.string_('bcd'), '0bc', numpy.float32(5.5)]), - ('(42;::;`foo)', [numpy.int64(42), None, numpy.string_('foo')]), - ('`the`quick`brown`fox', qlist(numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)), - ('``quick``fox', qlist(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)), - ('``', qlist(numpy.array([qnull(QSYMBOL), qnull(QSYMBOL)], dtype=numpy.object), qtype=QSYMBOL_LIST)), - ('("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")', - ['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog']), - ('("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")', - ['quick', ' ', 'fox', 'jumps', 'over', 'a lazy', 'dog']), - ('{x+y}', QLambda('{x+y}')), - ('{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)])), - ('insert [1]', QProjection([QFunction(0), numpy.int64(1)])), - ('xbar', QLambda('k){x*y div x:$[16h=abs[@x];"j"$x;x]}')), - ('not', QFunction(0)), - ('and', QFunction(0)), - ('md5', QProjection([QFunction(0), numpy.int64(-15)])), - ('any', QFunction(0)), - ('save', QFunction(0)), - ('raze', QFunction(0)), - ('sums', QFunction(0)), - ('prev', QFunction(0)), - - ('(enlist `a)!(enlist 1)', QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST), + (b'::', None), + (b'1+`', QException(b'type')), + (b'1', numpy.int64(1)), + (b'1i', numpy.int32(1)), + (b'-234h', numpy.int16(-234)), + (b'0b', numpy.bool_(False)), + (b'1b', numpy.bool_(True)), + (b'0x2a', numpy.byte(0x2a)), + (b'89421099511627575j', numpy.int64(long(89421099511627575))), + (b'5.5e', numpy.float32(5.5)), + (b'3.234', numpy.float64(3.234)), + (b'"0"', b'0'), + (b'"abc"', b'abc'), + (b'"quick brown fox jumps over a lazy dog"', b'quick brown fox jumps over a lazy dog'), + (b'`abc', numpy.string_('abc')), + (b'`quickbrownfoxjumpsoveralazydog', numpy.string_('quickbrownfoxjumpsoveralazydog')), + (b'0Nh', qnull(QSHORT)), + (b'0N', qnull(QLONG)), + (b'0Ni', qnull(QINT)), + (b'0Nj', qnull(QLONG)), + (b'0Ne', qnull(QFLOAT)), + (b'0n', qnull(QDOUBLE)), + (b'" "', qnull(QSTRING)), + (b'`', qnull(QSYMBOL)), + (b'0Ng', qnull(QGUID)), + (b'()', []), + (b'(0b;1b;0b)', qlist(numpy.array([False, True, False], dtype=numpy.bool_), qtype=QBOOL_LIST)), + (b'(0x01;0x02;0xff)', qlist(numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), qtype=QBYTE_LIST)), + (b'(1h;2h;3h)', qlist(numpy.array([1, 2, 3], dtype=numpy.int16), qtype=QSHORT_LIST)), + (b'(1h;0Nh;3h)', qlist(numpy.array([1, qnull(QSHORT), 3], dtype=numpy.int16), qtype=QSHORT_LIST)), + (b'1 2 3', qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST)), + (b'1 0N 3', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)), + (b'(1i;2i;3i)', qlist(numpy.array([1, 2, 3], dtype=numpy.int32), qtype=QINT_LIST)), + (b'(1i;0Ni;3i)', qlist(numpy.array([1, qnull(QINT), 3], dtype=numpy.int32), qtype=QINT_LIST)), + (b'(1j;2j;3j)', qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST)), + (b'(1j;0Nj;3j)', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)), + (b'(5.5e; 8.5e)', qlist(numpy.array([5.5, 8.5], dtype=numpy.float32), qtype=QFLOAT_LIST)), + (b'(5.5e; 0Ne)', qlist(numpy.array([5.5, qnull(QFLOAT)], dtype=numpy.float32), qtype=QFLOAT_LIST)), + (b'3.23 6.46', qlist(numpy.array([3.23, 6.46], dtype=numpy.float64), qtype=QDOUBLE_LIST)), + (b'3.23 0n', qlist(numpy.array([3.23, qnull(QDOUBLE)], dtype=numpy.float64), qtype=QDOUBLE_LIST)), + (b'(1;`bcd;"0bc";5.5e)', [numpy.int64(1), numpy.string_('bcd'), b'0bc', numpy.float32(5.5)]), + (b'(42;::;`foo)', [numpy.int64(42), None, numpy.string_('foo')]), + (b'`the`quick`brown`fox', qlist(numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)), + (b'``quick``fox', qlist(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)), + (b'``', qlist(numpy.array([qnull(QSYMBOL), qnull(QSYMBOL)], dtype=numpy.object), qtype=QSYMBOL_LIST)), + (b'("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")', + [b'quick', b'brown', b'fox', b'jumps', b'over', b'a lazy', b'dog']), + (b'("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")', + [b'quick', b' ', b'fox', b'jumps', b'over', b'a lazy', b'dog']), + (b'{x+y}', QLambda('{x+y}')), + (b'{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)])), + (b'insert [1]', QProjection([QFunction(0), numpy.int64(1)])), + (b'xbar', QLambda('k){x*y div x:$[16h=abs[@x];"j"$x;x]}')), + (b'not', QFunction(0)), + (b'and', QFunction(0)), + (b'md5', QProjection([QFunction(0), numpy.int64(-15)])), + (b'any', QFunction(0)), + (b'save', QFunction(0)), + (b'raze', QFunction(0)), + (b'sums', QFunction(0)), + (b'prev', QFunction(0)), + + (b'(enlist `a)!(enlist 1)', QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST), qlist(numpy.array([1], dtype=numpy.int64), qtype=QLONG_LIST))), - ('1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST), + (b'1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))), - ('`abc`def`gh!([] one: 1 2 3; two: 4 5 6)', QDictionary(qlist(numpy.array(['abc', 'def', 'gh']), qtype = QSYMBOL_LIST), + (b'`abc`def`gh!([] one: 1 2 3; two: 4 5 6)', QDictionary(qlist(numpy.array(['abc', 'def', 'gh']), qtype = QSYMBOL_LIST), qtable(qlist(numpy.array(['one', 'two']), qtype = QSYMBOL_LIST), [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)]))), - ('(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)], + (b'(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)], qlist(numpy.array(['first', 'second']), qtype = QSYMBOL_LIST))), - ('(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4'], - [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])), - ('`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST), - [[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), ['a', numpy.int64(2)]), numpy.float32(5.5)])), + (b'(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), b'4'], + [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), b'456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])), + (b'`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST), + [[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), [b'a', numpy.int64(2)]), numpy.float32(5.5)])), - ('flip `abc`def!(1 2 3; 4 5 6)', qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST), + (b'flip `abc`def!(1 2 3; 4 5 6)', qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST), [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)])), - ('flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)', + (b'flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)', qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST)])), - ('flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")', + (b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")', qtable(qlist(numpy.array(['name', 'iq', 'grade']), qtype = QSYMBOL_LIST), [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), - "a c"])), - ('flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))', + b"a c"])), + (b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))', qtable(qlist(numpy.array(['name', 'iq', 'fullname']), qtype = QSYMBOL_LIST), [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), - ["Arthur Dent", "Zaphod Beeblebrox", "Ford Prefect"]])), - ('flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; " "; "Ford Prefect"))', + [b"Arthur Dent", b"Zaphod Beeblebrox", b"Ford Prefect"]])), + (b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; " "; "Ford Prefect"))', qtable(qlist(numpy.array(['name', 'iq', 'fullname']), qtype = QSYMBOL_LIST), [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), - ["Arthur Dent", " ", "Ford Prefect"]])), - ('([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), + [b"Arthur Dent", b" ", b"Ford Prefect"]])), + (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), [qlist(numpy.array([1, 2]), qtype = QLONG_LIST), qlist(numpy.array([3, 4]), qtype = QLONG_LIST), qlist(numpy.array([5, 6, 7]), qtype = QLONG_LIST)]])), - ('([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), + (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), [qlist(numpy.array([1, 2]), qtype = QLONG_LIST), qlist(numpy.array([3, 4]), qtype = QLONG_LIST), qlist(numpy.array([5, 6]), qtype = QLONG_LIST)]])), - ('([] name:`symbol$(); iq:`int$())', qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), + (b'([] name:`symbol$(); iq:`int$())', qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), [qlist(numpy.array([], dtype=numpy.string_), qtype = QSYMBOL_LIST), qlist(numpy.array([]), qtype = QINT_LIST)])), - ('([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', + (b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST), [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)])), - ('([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', + (b'([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST), [qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]), qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST), @@ -185,31 +191,31 @@ NUMPY_TEMPORAL_EXPRESSIONS = OrderedDict(( - ('(2001.01m; 0Nm)', qlist(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), qtype=QMONTH_LIST)), - ('2001.01m', numpy.datetime64('2001-01', 'M')), - ('0Nm', numpy.datetime64('NaT', 'M')), - ('2001.01.01 2000.05.01 0Nd', qlist(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), qtype=QDATE_LIST)), - ('2001.01.01', numpy.datetime64('2001-01-01', 'D')), - ('0Nd', numpy.datetime64('NaT', 'D')), - ('2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST)), - ('2000.01.04T05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms')), - ('0Nz', numpy.datetime64('NaT', 'ms')), - ('12:01 0Nu', qlist(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype = QMINUTE)), - ('12:01', numpy.timedelta64(721, 'm')), - ('0Nu', numpy.timedelta64('NaT', 'm')), - ('12:05:00 0Nv', qlist(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype = QSECOND)), - ('12:05:00', numpy.timedelta64(43500, 's')), - ('0Nv', numpy.timedelta64('nat', 's')), - ('12:04:59.123 0Nt', qlist(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]), qtype = QTIME_LIST)), - ('12:04:59.123', numpy.timedelta64(43499123, 'ms')), - ('0Nt', numpy.timedelta64('NaT', 'ms')), - ('2000.01.04D05:36:57.600 0Np', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST)), - ('2000.01.04D05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns')), - ('0Np', numpy.datetime64('NaT', 'ns')), - ('0D05:36:57.600 0Nn', qlist(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype = QTIMESPAN_LIST)), - ('0D05:36:57.600', numpy.timedelta64(20217600000000, 'ns')), - ('0Nn', numpy.timedelta64('NaT', 'ns')), - ('([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', + (b'(2001.01m; 0Nm)', qlist(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), qtype=QMONTH_LIST)), + (b'2001.01m', numpy.datetime64('2001-01', 'M')), + (b'0Nm', numpy.datetime64('NaT', 'M')), + (b'2001.01.01 2000.05.01 0Nd', qlist(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), qtype=QDATE_LIST)), + (b'2001.01.01', numpy.datetime64('2001-01-01', 'D')), + (b'0Nd', numpy.datetime64('NaT', 'D')), + (b'2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST)), + (b'2000.01.04T05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms')), + (b'0Nz', numpy.datetime64('NaT', 'ms')), + (b'12:01 0Nu', qlist(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype = QMINUTE)), + (b'12:01', numpy.timedelta64(721, 'm')), + (b'0Nu', numpy.timedelta64('NaT', 'm')), + (b'12:05:00 0Nv', qlist(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype = QSECOND)), + (b'12:05:00', numpy.timedelta64(43500, 's')), + (b'0Nv', numpy.timedelta64('nat', 's')), + (b'12:04:59.123 0Nt', qlist(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]), qtype = QTIME_LIST)), + (b'12:04:59.123', numpy.timedelta64(43499123, 'ms')), + (b'0Nt', numpy.timedelta64('NaT', 'ms')), + (b'2000.01.04D05:36:57.600 0Np', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST)), + (b'2000.01.04D05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns')), + (b'0Np', numpy.datetime64('NaT', 'ns')), + (b'0D05:36:57.600 0Nn', qlist(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype = QTIMESPAN_LIST)), + (b'0D05:36:57.600', numpy.timedelta64(20217600000000, 'ns')), + (b'0Nn', numpy.timedelta64('NaT', 'ns')), + (b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', qtable(['pos', 'dates'], [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')])), @@ -218,10 +224,10 @@ COMPRESSED_EXPRESSIONS = OrderedDict(( - ('1000#`q', qlist(numpy.array(['q'] * 1000), qtype=QSYMBOL_LIST)), - ('([] q:1000#`q)', qtable(qlist(numpy.array(['q']), qtype = QSYMBOL_LIST), + (b'1000#`q', qlist(numpy.array(['q'] * 1000), qtype=QSYMBOL_LIST)), + (b'([] q:1000#`q)', qtable(qlist(numpy.array(['q']), qtype = QSYMBOL_LIST), [qlist(numpy.array(['q'] * 1000), qtype=QSYMBOL_LIST)])), - ('([] a:til 200;b:25+til 200;c:200#`a)', qtable(qlist(numpy.array(['a', 'b', 'c']), qtype = QSYMBOL_LIST), + (b'([] a:til 200;b:25+til 200;c:200#`a)', qtable(qlist(numpy.array(['a', 'b', 'c']), qtype = QSYMBOL_LIST), [qlist(numpy.arange(200), qtype=QLONG_LIST), qlist(numpy.arange(200) + 25, qtype=QLONG_LIST), qlist(numpy.array(['a'] * 200), qtype=QSYMBOL_LIST)])), @@ -233,23 +239,23 @@ def arrays_equal(left, right): if type(left) != type(right): return False - + if type(left) == numpy.ndarray and left.dtype != right.dtype: print('Type comparison failed: %s != %s' % (left.dtype, right.dtype)) return False - + if type(left) == QList and left.meta.qtype != right.meta.qtype: print('QType comparison failed: %s != %s' % (left.meta.qtype, right.meta.qtype)) return False - + if len(left) != len(right): return False - - for i in xrange(len(left)): + + for i in range(len(left)): if type(left[i]) != type(right[i]): print('Type comparison failed: %s != %s' % (type(left[i]), type(right[i]))) return False - + if not compare(left[i], right[i]): print('Value comparison failed: %s != %s' % ( left[i], right[i])) return False @@ -272,7 +278,7 @@ def compare(left, right): def test_reading(): BINARY = OrderedDict() - + with open('tests/QExpressions3.out', 'rb') as f: while True: query = f.readline().strip() @@ -285,40 +291,40 @@ def test_reading(): buffer_reader = qreader.QReader(None) print('Deserialization') - for query, value in EXPRESSIONS.iteritems(): - buffer_ = cStringIO.StringIO() + for query, value in iter(EXPRESSIONS.items()): + buffer_ = BytesIO() binary = binascii.unhexlify(BINARY[query]) - - buffer_.write('\1\0\0\0') + + buffer_.write(b'\1\0\0\0') buffer_.write(struct.pack('i', len(binary) + 8)) buffer_.write(binary) buffer_.seek(0) - + sys.stdout.write( ' %-75s' % query ) try: header = buffer_reader.read_header(source = buffer_.getvalue()) result = buffer_reader.read_data(message_size = header.size, is_compressed = header.is_compressed, raw = True) assert compare(buffer_.getvalue()[8:], result), 'raw reading failed: %s' % (query) - + stream_reader = qreader.QReader(buffer_) result = stream_reader.read(raw = True).data assert compare(buffer_.getvalue()[8:], result), 'raw reading failed: %s' % (query) result = buffer_reader.read(source = buffer_.getvalue()).data assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result) - + header = buffer_reader.read_header(source = buffer_.getvalue()) result = buffer_reader.read_data(message_size = header.size, is_compressed = header.is_compressed) assert compare(value, result), 'deserialization failed: %s' % (query) - + buffer_.seek(0) stream_reader = qreader.QReader(buffer_) result = stream_reader.read().data assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result) print('.') - except QException, e: + except QException as e: assert isinstance(value, QException) - assert e.message == value.message + assert e.args == value.args print('.') @@ -337,15 +343,15 @@ def test_reading_numpy_temporals(): BINARY[query] = binary print('Deserialization (numpy temporals)') - for query, value in NUMPY_TEMPORAL_EXPRESSIONS.iteritems(): - buffer_ = cStringIO.StringIO() + for query, value in iter(NUMPY_TEMPORAL_EXPRESSIONS.items()): + buffer_ = BytesIO() binary = binascii.unhexlify(BINARY[query]) - - buffer_.write('\1\0\0\0') + + buffer_.write(b'\1\0\0\0') buffer_.write(struct.pack('i', len(binary) + 8)) buffer_.write(binary) buffer_.seek(0) - + sys.stdout.write( ' %-75s' % query ) try: buffer_.seek(0) @@ -353,16 +359,16 @@ def test_reading_numpy_temporals(): result = stream_reader.read(numpy_temporals = True).data assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result) print('.') - except QException, e: + except QException as e: assert isinstance(value, QException) - assert e.message == value.message + assert e.args == value.args print('.') def test_reading_compressed(): BINARY = OrderedDict() - + with open('tests/QCompressedExpressions3.out', 'rb') as f: while True: query = f.readline().strip() @@ -372,18 +378,18 @@ def test_reading_compressed(): break BINARY[query] = binary - - print('Compressed deserialization') + + print('Compressed deserialization') buffer_reader = qreader.QReader(None) - for query, value in COMPRESSED_EXPRESSIONS.iteritems(): - buffer_ = cStringIO.StringIO() + for query, value in iter(COMPRESSED_EXPRESSIONS.items()): + buffer_ = BytesIO() binary = binascii.unhexlify(BINARY[query]) - - buffer_.write('\1\0\1\0') + + buffer_.write(b'\1\0\1\0') buffer_.write(struct.pack('i', len(binary) + 8)) buffer_.write(binary) buffer_.seek(0) - + sys.stdout.write( ' %-75s' % query ) try: result = buffer_reader.read(source = buffer_.getvalue()).data @@ -397,12 +403,12 @@ def test_reading_compressed(): result = stream_reader.read().data assert compare(value, result), 'deserialization failed: %s' % (query) print('.') - except QException, e: + except QException as e: assert isinstance(value, QException) - assert e.message == value.message + assert e.args == value.args print('.') - + test_reading() test_reading_numpy_temporals() diff --git a/tests/qtypes_test.py b/tests/qtypes_test.py index 66e83d2..387b129 100644 --- a/tests/qtypes_test.py +++ b/tests/qtypes_test.py @@ -15,6 +15,10 @@ # import pytest +import sys + +if sys.version > '3': + long = int from qpython.qtype import * # @UnusedWildImport from qpython.qcollection import * # @UnusedWildImport @@ -25,20 +29,20 @@ def test_is_null(): assert is_null(qnull(QSYMBOL), QSYMBOL) assert is_null(numpy.string_(''), QSYMBOL) - assert is_null('', QSYMBOL) - assert not is_null(' ', QSYMBOL) + assert is_null(b'', QSYMBOL) + assert not is_null(b' ', QSYMBOL) assert not is_null(numpy.string_(' '), QSYMBOL) - + assert is_null(qnull(QSTRING), QSTRING) - assert is_null(' ', QSTRING) - assert not is_null('', QSTRING) + assert is_null(b' ', QSTRING) + assert not is_null(b'', QSTRING) assert not is_null(numpy.string_(''), QSTRING) assert is_null(numpy.string_(' '), QSTRING) - + assert is_null(qnull(QBOOL), QBOOL) assert is_null(numpy.bool_(False), QBOOL) assert not is_null(numpy.bool_(True), QBOOL) - + for t in QNULLMAP.keys(): assert is_null(qnull(t), t) @@ -46,20 +50,20 @@ def test_is_null(): def test_qdict(): with pytest.raises(ValueError): - QDictionary(qlist(numpy.array([1, 2, 3]), qtype=QLONG_LIST), qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST)) - + QDictionary(qlist(numpy.array([1, 2, 3]), qtype=QLONG_LIST), qlist(numpy.array(['abc', 'cdefgh']), qtype=QSYMBOL_LIST)) + d = QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST), - qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST)) - + qlist(numpy.array(['abc', 'cdefgh']), qtype=QSYMBOL_LIST)) + assert len(d) == 2 - assert d[1] == 'abc' + assert d[1] == b'abc' with pytest.raises(KeyError): d['abc'] assert numpy.int64(1) in d assert not -1 in d - + i = 0 for k in d: assert k == d.keys[i] @@ -70,18 +74,18 @@ def test_qdict(): for kv in d.iteritems(): assert kv == (d.keys[i], d.values[i]) i += 1 - + i = 0 for kv in d.items(): assert kv == (d.keys[i], d.values[i]) i += 1 - + i = 0 for k in d.iterkeys(): assert k == d.keys[i] assert d[k] == d.values[i] i += 1 - + i = 0 for v in d.itervalues(): assert v == d.values[i] @@ -90,50 +94,50 @@ def test_qdict(): def test_qtable(): with pytest.raises(ValueError): - qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST), - [qlist([], qtype = QSYMBOL_LIST)]) - - with pytest.raises(ValueError): + qtable(qlist(['name', 'iq'], qtype=QSYMBOL_LIST), + [qlist([], qtype=QSYMBOL_LIST)]) + + with pytest.raises(ValueError): qtable(['name', 'iq'], - [['Beeblebrox', 'Prefect'], + [['Beeblebrox', 'Prefect'], [98, 42, 126]], - name = QSYMBOL, iq = QLONG) - + name=QSYMBOL, iq=QLONG) + t = qtable(['name', 'iq'], - [['Dent', 'Beeblebrox', 'Prefect'], + [['Dent', 'Beeblebrox', 'Prefect'], [98, 42, 126]], - name = QSYMBOL, iq = QLONG) - + name=QSYMBOL, iq=QLONG) + assert len(t) == 3 - assert t[t['name'] == 'Dent']['name'] == 'Dent' - assert t[t['name'] == 'Dent']['iq'] == 98L - + assert t[t['name'] == 'Dent']['name'] == b'Dent' + assert t[t['name'] == 'Dent']['iq'] == long(98) + def test_qkeyedtable(): with pytest.raises(ValueError): - QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]), + QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype=QSYMBOL_LIST), + [qlist(numpy.array([1001, 1002, 1003]), qtype=QLONG_LIST)]), ()) - + with pytest.raises(ValueError): QKeyedTable((), - qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), - qlist(numpy.array([366, 121, qnull(QDATE)]), qtype = QDATE_LIST)])) - - t = QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]), - qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), - qlist(numpy.array([366, 121, 255]), qtype = QINT_LIST)])) - + qtable(qlist(numpy.array(['pos', 'dates']), qtype=QSYMBOL_LIST), + [qlist(numpy.array(['d1', 'd2', 'd3']), qtype=QSYMBOL_LIST), + qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)])) + + t = QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype=QSYMBOL_LIST), + [qlist(numpy.array([1001, 1002, 1003]), qtype=QLONG_LIST)]), + qtable(qlist(numpy.array(['pos', 'dates']), qtype=QSYMBOL_LIST), + [qlist(numpy.array(['d1', 'd2', 'd3']), qtype=QSYMBOL_LIST), + qlist(numpy.array([366, 121, 255]), qtype=QINT_LIST)])) + assert len(t) == 3 - assert t.keys[t.keys['eid'] == 1002]['eid'] == 1002L - assert t.values[t.keys['eid'] == 1002]['pos'][0] == 'd2' + assert t.keys[t.keys['eid'] == 1002]['eid'] == long(1002) + assert t.values[t.keys['eid'] == 1002]['pos'][0] == b'd2' assert t.values[t.keys['eid'] == 1002]['dates'][0] == 121 - + i = 0 for k in t: assert k == t.keys[i] @@ -143,32 +147,32 @@ def test_qkeyedtable(): for kv in t.iteritems(): assert kv == (t.keys[i], t.values[i]) i += 1 - + i = 0 for kv in t.items(): assert kv == (t.keys[i], t.values[i]) i += 1 - + i = 0 for k in t.iterkeys(): assert k == t.keys[i] i += 1 - + i = 0 for v in t.itervalues(): assert v == t.values[i] i += 1 - + def test_qtemporallist(): na_dt = numpy.arange('1999-01-01', '2005-12-31', dtype='datetime64[D]') - na = array_to_raw_qtemporal(na_dt, qtype = QDATE_LIST) + na = array_to_raw_qtemporal(na_dt, qtype=QDATE_LIST) t = qlist(na, qtype=QDATE_LIST) - + assert t.meta.qtype == -abs(QDATE_LIST) - - for x in xrange(len(na)): + + for x in range(len(na)): assert t.raw(x) == x - 365 assert t[x].raw == na_dt[x] x += 1 @@ -176,147 +180,147 @@ def test_qtemporallist(): def test_array_to_raw_qtemporal(): na_dt = numpy.arange('1999-01', '2005-12', dtype='datetime64[M]') - na = array_to_raw_qtemporal(na_dt, qtype = QMONTH_LIST) + na = array_to_raw_qtemporal(na_dt, qtype=QMONTH_LIST) assert na.dtype == numpy.int32 - - for x in xrange(len(na)): + + for x in range(len(na)): assert na[x] == x - 12 x += 1 - + na_dt = numpy.arange('1999-01-01', '2005-12-31', dtype='datetime64[D]') - na = array_to_raw_qtemporal(na_dt, qtype = QDATE_LIST) + na = array_to_raw_qtemporal(na_dt, qtype=QDATE_LIST) assert na.dtype == numpy.int32 - - for x in xrange(len(na)): + + for x in range(len(na)): assert na[x] == x - 365 x += 1 - + na_dt = numpy.arange('1999-01-01T00:00:00.000Z', '2001-01-04T05:36:57.600Z', 12345678, dtype='datetime64[ms]') - na = array_to_raw_qtemporal(na_dt, qtype = QDATETIME_LIST) + na = array_to_raw_qtemporal(na_dt, qtype=QDATETIME_LIST) assert na.dtype == numpy.float64 - + step = 12346678. / _MILLIS_PER_DAY - + assert na[0] == -365.0 assert abs(na[-1] - 369.1677) < 0.001 - - for x in xrange(len(na)): + + for x in range(len(na)): ref = (x * step) - 365 assert abs(na[x] - ref) < 0.1, '%s %s' %(na[x], ref) - + na_dt = numpy.arange('1999-01-01T00:00:00.000Z', '2001-01-04T05:36:57.600Z', 1234567890000, dtype='datetime64[ns]') - na = array_to_raw_qtemporal(na_dt, qtype = QTIMESTAMP_LIST) + na = array_to_raw_qtemporal(na_dt, qtype=QTIMESTAMP_LIST) assert na.dtype == numpy.int64 - - ref = -31536000000000000L - for x in xrange(len(na)): + + ref = long(-31536000000000000) + for x in range(len(na)): assert na[x] == ref - ref += 1234567890000L - + ref += long(1234567890000) + na_dt = numpy.arange(-1000000, 1000000, 12345, dtype='timedelta64[m]') - na = array_to_raw_qtemporal(na_dt, qtype = QMINUTE) + na = array_to_raw_qtemporal(na_dt, qtype=QMINUTE) assert na.dtype == numpy.int32 - for x in xrange(len(na)): + for x in range(len(na)): assert na[x] == -1000000 + x * 12345 - + na_dt = numpy.arange(-1000000, 1000000, 12345, dtype='timedelta64[ms]') - na = array_to_raw_qtemporal(na_dt, qtype = QTIME) + na = array_to_raw_qtemporal(na_dt, qtype=QTIME) assert na.dtype == numpy.int32 - for x in xrange(len(na)): + for x in range(len(na)): assert na[x] == -1000000 + x * 12345 - + na_dt = numpy.arange(-1000000, 1000000, 12345, dtype='timedelta64[s]') - na = array_to_raw_qtemporal(na_dt, qtype = QSECOND) + na = array_to_raw_qtemporal(na_dt, qtype=QSECOND) assert na.dtype == numpy.int32 - for x in xrange(len(na)): + for x in range(len(na)): assert na[x] == -1000000 + x * 12345 - + na_dt = numpy.arange(-1000000, 1000000, 12345, dtype='timedelta64[ns]') - na = array_to_raw_qtemporal(na_dt, qtype = QTIMESPAN) + na = array_to_raw_qtemporal(na_dt, qtype=QTIMESPAN) assert na.dtype == numpy.int64 - for x in xrange(len(na)): + for x in range(len(na)): assert na[x] == -1000000 + x * 12345 - + def test_array_from_raw_qtemporal(): raw = numpy.array([12, 121, qnull(QMONTH)]) - na_dt = array_from_raw_qtemporal(raw, qtype = QMONTH) - + na_dt = array_from_raw_qtemporal(raw, qtype=QMONTH) + assert str(na_dt.dtype).startswith('datetime64[M]') - for x in xrange(len(na_dt)): + for x in range(len(na_dt)): if na_dt[x] != numpy.datetime64('NaT', 'M'): assert na_dt[x].astype(int) == raw[x] + 360 else: assert raw[x] == qnull(QMONTH) - + raw = numpy.array([366, 121, qnull(QDATE)]) - na_dt = array_from_raw_qtemporal(raw, qtype = QDATE) - + na_dt = array_from_raw_qtemporal(raw, qtype=QDATE) + assert str(na_dt.dtype).startswith('datetime64[D]') - for x in xrange(len(na_dt)): + for x in range(len(na_dt)): if na_dt[x] != numpy.datetime64('NaT', 'D'): assert na_dt[x].astype(int) == raw[x] + 10957 else: - assert raw[x] == qnull(QDATE) - + assert raw[x] == qnull(QDATE) + raw = numpy.array([43500, -121, qnull(QMINUTE)]) - na_dt = array_from_raw_qtemporal(raw, qtype = QMINUTE) - + na_dt = array_from_raw_qtemporal(raw, qtype=QMINUTE) + assert str(na_dt.dtype).startswith('timedelta64[m]') - for x in xrange(len(na_dt)): + for x in range(len(na_dt)): if na_dt[x] != numpy.timedelta64('NaT', 'm'): assert na_dt[x].astype(int) == raw[x] else: assert raw[x] == qnull(QMINUTE) - + raw = numpy.array([43500, -121, qnull(QSECOND)]) - na_dt = array_from_raw_qtemporal(raw, qtype = QSECOND) - + na_dt = array_from_raw_qtemporal(raw, qtype=QSECOND) + assert str(na_dt.dtype).startswith('timedelta64[s]') - for x in xrange(len(na_dt)): + for x in range(len(na_dt)): if na_dt[x] != numpy.timedelta64('NaT', 's'): assert na_dt[x].astype(int) == raw[x] else: assert raw[x] == qnull(QSECOND) - + raw = numpy.array([43500, -121, qnull(QTIME)]) - na_dt = array_from_raw_qtemporal(raw, qtype = QTIME) - + na_dt = array_from_raw_qtemporal(raw, qtype=QTIME) + assert str(na_dt.dtype).startswith('timedelta64[ms]') - for x in xrange(len(na_dt)): + for x in range(len(na_dt)): if na_dt[x] != numpy.timedelta64('NaT', 'ms'): assert na_dt[x].astype(int) == raw[x] else: assert raw[x] == qnull(QTIME) - + raw = numpy.array([20217600000000, -2021760000, qnull(QTIMESPAN)]) - na_dt = array_from_raw_qtemporal(raw, qtype = QTIMESPAN) - + na_dt = array_from_raw_qtemporal(raw, qtype=QTIMESPAN) + assert str(na_dt.dtype).startswith('timedelta64[ns]') - for x in xrange(len(na_dt)): + for x in range(len(na_dt)): if na_dt[x] != numpy.timedelta64('NaT', 'ns'): assert na_dt[x].astype(numpy.int64) == raw[x] else: assert raw[x] == qnull(QTIMESPAN) raw = numpy.array([279417600000000, -2021760000, qnull(QTIMESTAMP)]) - na_dt = array_from_raw_qtemporal(raw, qtype = QTIMESTAMP) - + na_dt = array_from_raw_qtemporal(raw, qtype=QTIMESTAMP) + assert str(na_dt.dtype).startswith('datetime64[ns]') - for x in xrange(len(na_dt)): + for x in range(len(na_dt)): if na_dt[x] != numpy.datetime64('NaT', 'ns'): - assert na_dt[x].astype(numpy.int64) == raw[x] + numpy.datetime64('2000-01-01T00:00:00Z', 'ns').astype(long) + assert na_dt[x].astype(numpy.int64) == raw[x] + numpy.datetime64('2000-01-01T00:00:00Z', 'ns').astype(numpy.int64) else: assert raw[x] == qnull(QTIMESTAMP) - - + + raw = numpy.array([3.234, qnull(QDATETIME)]) - na_dt = array_from_raw_qtemporal(raw, qtype = QDATETIME) + na_dt = array_from_raw_qtemporal(raw, qtype=QDATETIME) ref = numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]) - + assert str(na_dt.dtype).startswith('datetime64[ms]') - for x in xrange(len(na_dt)): + for x in range(len(na_dt)): assert na_dt[x] == ref[x] @@ -326,4 +330,4 @@ def test_array_from_raw_qtemporal(): test_qkeyedtable() test_qtemporallist() test_array_to_raw_qtemporal() -test_array_from_raw_qtemporal() \ No newline at end of file +test_array_from_raw_qtemporal() diff --git a/tests/qwriter_test.py b/tests/qwriter_test.py index 1d243d3..3fe197f 100644 --- a/tests/qwriter_test.py +++ b/tests/qwriter_test.py @@ -16,6 +16,8 @@ import binascii import sys +if sys.version > '3': + long = int from collections import OrderedDict from qpython import qwriter @@ -26,203 +28,207 @@ BINARY = OrderedDict() EXPRESSIONS = OrderedDict(( - ('("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)', + (b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")', + qtable(qlist(numpy.array(['name', 'iq', 'grade']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), + qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), + "a c"])), + (b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)', qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST)), - ('"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')), - ('"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')), - ('(2001.01m; 0Nm)', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01', 'M'), QMONTH), qnull(QMONTH)]), qtype=QMONTH_LIST), + (b'"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')), + (b'"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')), + (b'(2001.01m; 0Nm)', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01', 'M'), QMONTH), qnull(QMONTH)]), qtype=QMONTH_LIST), qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST), qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2001-01', 'M'), numpy.datetime64('NaT', 'M')]), qtype = QMONTH_LIST), qtype = QMONTH_LIST), qlist([12, qnull(QMONTH)], qtype=QMONTH_LIST), qlist(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), qtype=QMONTH_LIST), numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), )), - ('2001.01m', (qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH), + (b'2001.01m', (qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH), numpy.datetime64('2001-01', 'M'))), - ('0Nm', (qtemporal(qnull(QMONTH), qtype=QMONTH), + (b'0Nm', (qtemporal(qnull(QMONTH), qtype=QMONTH), qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH), numpy.datetime64('NaT', 'M'))), - ('2001.01.01 2000.05.01 0Nd', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), to_raw_qtemporal(numpy.datetime64('2000-05-01', 'D'), qtype=QDATE), qnull(QDATE)]), qtype=QDATE_LIST), + (b'2001.01.01 2000.05.01 0Nd', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), to_raw_qtemporal(numpy.datetime64('2000-05-01', 'D'), qtype=QDATE), qnull(QDATE)]), qtype=QDATE_LIST), qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST), qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2001-01-01', 'D'), numpy.datetime64('2000-05-01', 'D'), numpy.datetime64('NaT', 'D')]), qtype = QDATE_LIST), qtype = QDATE_LIST), qlist([366, 121, qnull(QDATE)], qtype=QDATE_LIST), qlist(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), qtype=QDATE_LIST), numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), )), - ('2001.01.01', (qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), + (b'2001.01.01', (qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), numpy.datetime64('2001-01-01', 'D'))), - ('0Nd', (qtemporal(qnull(QDATE), qtype=QDATE), + (b'0Nd', (qtemporal(qnull(QDATE), qtype=QDATE), qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE), numpy.datetime64('NaT', 'D'))), - ('2000.01.04T05:36:57.600 0Nz', (qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST), + (b'2000.01.04T05:36:57.600 0Nz', (qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST), qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]), qtype=QDATETIME_LIST), qtype=QDATETIME_LIST), qlist([3.234, qnull(QDATETIME)], qtype=QDATETIME_LIST), qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST), numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]) )), - ('2000.01.04T05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), qtype=QDATETIME), + (b'2000.01.04T05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), qtype=QDATETIME), numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'))), - ('0Nz', (qtemporal(qnull(QDATETIME), qtype=QDATETIME), + (b'0Nz', (qtemporal(qnull(QDATETIME), qtype=QDATETIME), qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME), numpy.datetime64('NaT', 'ms'))), - ('12:01 0Nu', (qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST), + (b'12:01 0Nu', (qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST), qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype=QMINUTE_LIST), qtype=QMINUTE_LIST), qlist([721, qnull(QMINUTE)], qtype=QMINUTE_LIST), qlist(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype = QMINUTE), numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), )), - ('12:01', (qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE), + (b'12:01', (qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE), numpy.timedelta64(721, 'm'))), - ('0Nu', (qtemporal(qnull(QMINUTE), qtype=QMINUTE), + (b'0Nu', (qtemporal(qnull(QMINUTE), qtype=QMINUTE), qtemporal(numpy.timedelta64('NaT', 'm'), qtype=QMINUTE), numpy.timedelta64('NaT', 'm'))), - ('12:05:00 0Nv', (qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST), + (b'12:05:00 0Nv', (qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST), qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype=QSECOND_LIST), qtype=QSECOND_LIST), qlist([43500, qnull(QSECOND)], qtype=QSECOND_LIST), qlist(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype = QSECOND), numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]) )), - ('12:05:00', (qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND), + (b'12:05:00', (qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND), numpy.timedelta64(43500, 's'))), - ('0Nv', (qtemporal(qnull(QSECOND), qtype=QSECOND), + (b'0Nv', (qtemporal(qnull(QSECOND), qtype=QSECOND), qtemporal(numpy.timedelta64('nat', 's'), qtype=QSECOND), numpy.timedelta64('nat', 's'))), - ('12:04:59.123 0Nt', (qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST), + (b'12:04:59.123 0Nt', (qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST), qlist([43499123, qnull(QTIME)], qtype=QTIME_LIST), qlist(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]), qtype = QTIME_LIST), numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]) )), - ('12:04:59.123', (qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME), + (b'12:04:59.123', (qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME), numpy.timedelta64(43499123, 'ms'))), - ('0Nt', (qtemporal(qnull(QTIME), qtype=QTIME), + (b'0Nt', (qtemporal(qnull(QTIME), qtype=QTIME), qtemporal(numpy.timedelta64('NaT', 'ms'), qtype=QTIME), numpy.timedelta64('NaT', 'ms'))), - ('2000.01.04D05:36:57.600 0Np', (qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST), + (b'2000.01.04D05:36:57.600 0Np', (qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST), qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]), qtype=QTIMESTAMP_LIST), qtype=QTIMESTAMP_LIST), qlist([long(279417600000000), qnull(QTIMESTAMP)], qtype=QTIMESTAMP_LIST), qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST), numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]) )), - ('2000.01.04D05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), qtype=QTIMESTAMP), + (b'2000.01.04D05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), qtype=QTIMESTAMP), numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'))), - ('0Np', (qtemporal(qnull(QTIMESTAMP), qtype=QTIMESTAMP), + (b'0Np', (qtemporal(qnull(QTIMESTAMP), qtype=QTIMESTAMP), qtemporal(numpy.datetime64('NaT', 'ns'), qtype=QTIMESTAMP), numpy.datetime64('NaT', 'ns'))), - ('0D05:36:57.600 0Nn', (qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST), + (b'0D05:36:57.600 0Nn', (qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST), qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype=QTIMESPAN_LIST), qtype=QTIMESPAN_LIST), qlist([long(20217600000000), qnull(QTIMESPAN)], qtype=QTIMESPAN_LIST), qlist(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype = QTIMESPAN_LIST), numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]) )), - ('0D05:36:57.600', (qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN), + (b'0D05:36:57.600', (qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN), numpy.timedelta64(20217600000000, 'ns'))), - ('0Nn', (qtemporal(qnull(QTIMESPAN), qtype=QTIMESPAN), + (b'0Nn', (qtemporal(qnull(QTIMESPAN), qtype=QTIMESPAN), qtemporal(numpy.timedelta64('NaT', 'ns'), qtype=QTIMESPAN), numpy.timedelta64('NaT', 'ns'))), - ('::', None), - ('1+`', QException('type')), - ('1', numpy.int64(1)), - ('1i', numpy.int32(1)), - ('-234h', numpy.int16(-234)), - ('0b', numpy.bool_(False)), - ('1b', numpy.bool_(True)), - ('0x2a', numpy.byte(0x2a)), - ('89421099511627575j', numpy.int64(89421099511627575L)), - ('5.5e', numpy.float32(5.5)), - ('3.234', numpy.float64(3.234)), - ('"0"', '0'), - ('"abc"', ('abc', - numpy.array(list('abc')))), - ('"quick brown fox jumps over a lazy dog"', 'quick brown fox jumps over a lazy dog'), - ('`abc', numpy.string_('abc')), - ('`quickbrownfoxjumpsoveralazydog', numpy.string_('quickbrownfoxjumpsoveralazydog')), - ('0Nh', qnull(QSHORT)), - ('0N', qnull(QLONG)), - ('0Ni', qnull(QINT)), - ('0Nj', qnull(QLONG)), - ('0Ne', qnull(QFLOAT)), - ('0n', qnull(QDOUBLE)), - ('" "', qnull(QSTRING)), - ('`', qnull(QSYMBOL)), - ('0Ng', qnull(QGUID)), - ('()', []), - ('(0b;1b;0b)', (numpy.array([False, True, False], dtype=numpy.bool_), + (b'::', None), + (b'1+`', QException('type')), + (b'1', numpy.int64(1)), + (b'1i', numpy.int32(1)), + (b'-234h', numpy.int16(-234)), + (b'0b', numpy.bool_(False)), + (b'1b', numpy.bool_(True)), + (b'0x2a', numpy.byte(0x2a)), + (b'89421099511627575j', numpy.int64(long(89421099511627575))), + (b'5.5e', numpy.float32(5.5)), + (b'3.234', numpy.float64(3.234)), + (b'"0"', '0'), + (b'"abc"', ('abc', + numpy.array(list('abc'), dtype='S'))), + (b'"quick brown fox jumps over a lazy dog"', 'quick brown fox jumps over a lazy dog'), + (b'`abc', numpy.string_('abc')), + (b'`quickbrownfoxjumpsoveralazydog', numpy.string_('quickbrownfoxjumpsoveralazydog')), + (b'0Nh', qnull(QSHORT)), + (b'0N', qnull(QLONG)), + (b'0Ni', qnull(QINT)), + (b'0Nj', qnull(QLONG)), + (b'0Ne', qnull(QFLOAT)), + (b'0n', qnull(QDOUBLE)), + (b'" "', qnull(QSTRING)), + (b'`', qnull(QSYMBOL)), + (b'0Ng', qnull(QGUID)), + (b'()', []), + (b'(0b;1b;0b)', (numpy.array([False, True, False], dtype=numpy.bool_), qlist(numpy.array([False, True, False]), qtype = QBOOL_LIST), qlist([False, True, False], qtype = QBOOL_LIST))), - ('(0x01;0x02;0xff)', (numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), + (b'(0x01;0x02;0xff)', (numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), qlist(numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), qtype = QBYTE_LIST), qlist(numpy.array([0x01, 0x02, 0xff]), qtype = QBYTE_LIST), qlist([0x01, 0x02, 0xff], qtype = QBYTE_LIST))), - ('(1h;2h;3h)', (numpy.array([1, 2, 3], dtype=numpy.int16), + (b'(1h;2h;3h)', (numpy.array([1, 2, 3], dtype=numpy.int16), qlist(numpy.array([1, 2, 3], dtype=numpy.int16), qtype = QSHORT_LIST), qlist(numpy.array([1, 2, 3]), qtype = QSHORT_LIST), qlist([1, 2, 3], qtype = QSHORT_LIST))), - ('(1h;0Nh;3h)', qlist(numpy.array([1, qnull(QSHORT), 3], dtype=numpy.int16), qtype=QSHORT_LIST)), - ('1 2 3', (numpy.array([1, 2, 3], dtype=numpy.int64), + (b'(1h;0Nh;3h)', qlist(numpy.array([1, qnull(QSHORT), 3], dtype=numpy.int16), qtype=QSHORT_LIST)), + (b'1 2 3', (numpy.array([1, 2, 3], dtype=numpy.int64), qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST), qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), qlist([1, 2, 3], qtype = QLONG_LIST))), - ('1 0N 3', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)), - ('(1i;2i;3i)', (numpy.array([1, 2, 3], dtype=numpy.int32), + (b'1 0N 3', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)), + (b'(1i;2i;3i)', (numpy.array([1, 2, 3], dtype=numpy.int32), qlist(numpy.array([1, 2, 3], dtype=numpy.int32), qtype = QINT_LIST), qlist(numpy.array([1, 2, 3]), qtype = QINT_LIST), qlist([1, 2, 3], qtype = QINT_LIST))), - ('(1i;0Ni;3i)', qlist(numpy.array([1, qnull(QINT), 3], dtype=numpy.int32), qtype=QINT_LIST)), - ('(1j;2j;3j)', (numpy.array([1, 2, 3], dtype=numpy.int64), + (b'(1i;0Ni;3i)', qlist(numpy.array([1, qnull(QINT), 3], dtype=numpy.int32), qtype=QINT_LIST)), + (b'(1j;2j;3j)', (numpy.array([1, 2, 3], dtype=numpy.int64), qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST), qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), qlist([1, 2, 3], qtype = QLONG_LIST))), - ('(1j;0Nj;3j)', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)), - ('(5.5e; 8.5e)', (numpy.array([5.5, 8.5], dtype=numpy.float32), + (b'(1j;0Nj;3j)', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)), + (b'(5.5e; 8.5e)', (numpy.array([5.5, 8.5], dtype=numpy.float32), qlist(numpy.array([5.5, 8.5], dtype=numpy.float32), qtype = QFLOAT_LIST), qlist(numpy.array([5.5, 8.5]), qtype = QFLOAT_LIST), qlist([5.5, 8.5], qtype = QFLOAT_LIST))), - ('(5.5e; 0Ne)', qlist(numpy.array([5.5, qnull(QFLOAT)], dtype=numpy.float32), qtype=QFLOAT_LIST)), - ('3.23 6.46', (numpy.array([3.23, 6.46], dtype=numpy.float64), + (b'(5.5e; 0Ne)', qlist(numpy.array([5.5, qnull(QFLOAT)], dtype=numpy.float32), qtype=QFLOAT_LIST)), + (b'3.23 6.46', (numpy.array([3.23, 6.46], dtype=numpy.float64), qlist(numpy.array([3.23, 6.46], dtype=numpy.float64), qtype = QDOUBLE_LIST), qlist(numpy.array([3.23, 6.46]), qtype = QDOUBLE_LIST), qlist([3.23, 6.46], qtype = QDOUBLE_LIST))), - ('3.23 0n', qlist(numpy.array([3.23, qnull(QDOUBLE)], dtype=numpy.float64), qtype=QDOUBLE_LIST)), - ('(1;`bcd;"0bc";5.5e)', [numpy.int64(1), numpy.string_('bcd'), '0bc', numpy.float32(5.5)]), - ('(42;::;`foo)', [numpy.int64(42), None, numpy.string_('foo')]), - ('(1;2h;3.234;"4")', [numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4']), - ('(`one;2 3;"456";(7;8 9))', [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]]), + (b'3.23 0n', qlist(numpy.array([3.23, qnull(QDOUBLE)], dtype=numpy.float64), qtype=QDOUBLE_LIST)), + (b'(1;`bcd;"0bc";5.5e)', [numpy.int64(1), numpy.string_('bcd'), '0bc', numpy.float32(5.5)]), + (b'(42;::;`foo)', [numpy.int64(42), None, numpy.string_('foo')]), + (b'(1;2h;3.234;"4")', [numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4']), + (b'(`one;2 3;"456";(7;8 9))', [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]]), - ('`jumps`over`a`lazy`dog', (numpy.array(['jumps', 'over', 'a', 'lazy', 'dog'], dtype=numpy.string_), - numpy.array(['jumps', 'over', 'a', 'lazy', 'dog']), + (b'`jumps`over`a`lazy`dog', (numpy.array(['jumps', 'over', 'a', 'lazy', 'dog'], dtype=numpy.string_), qlist(numpy.array(['jumps', 'over', 'a', 'lazy', 'dog']), qtype = QSYMBOL_LIST), qlist(['jumps', 'over', 'a', 'lazy', 'dog'], qtype = QSYMBOL_LIST))), - ('`the`quick`brown`fox', numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')], dtype=numpy.object)), - ('``quick``fox', qlist(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)), - ('``', qlist(numpy.array([qnull(QSYMBOL), qnull(QSYMBOL)], dtype=numpy.object), qtype=QSYMBOL_LIST)), - ('("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")', + (b'`the`quick`brown`fox', numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')], dtype=numpy.object)), + (b'``quick``fox', qlist(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)), + (b'``', qlist(numpy.array([qnull(QSYMBOL), qnull(QSYMBOL)], dtype=numpy.object), qtype=QSYMBOL_LIST)), + (b'("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")', (['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog'], qlist(numpy.array(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog']), qtype = QSTRING_LIST), qlist(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog'], qtype = QSTRING_LIST))), - ('{x+y}', QLambda('{x+y}')), - ('{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)])), + (b'{x+y}', QLambda('{x+y}')), + (b'{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)])), - ('(enlist `a)!(enlist 1)', (QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST), + (b'(enlist `a)!(enlist 1)', (QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST), qlist(numpy.array([1], dtype=numpy.int64), qtype=QLONG_LIST)), QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST), qlist(numpy.array([1]), qtype=QLONG_LIST)))), - ('1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST), + (b'1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))), - ('`abc`def`gh!([] one: 1 2 3; two: 4 5 6)', QDictionary(qlist(numpy.array(['abc', 'def', 'gh']), qtype = QSYMBOL_LIST), + (b'`abc`def`gh!([] one: 1 2 3; two: 4 5 6)', QDictionary(qlist(numpy.array(['abc', 'def', 'gh']), qtype = QSYMBOL_LIST), qtable(qlist(numpy.array(['one', 'two']), qtype = QSYMBOL_LIST), [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)]))), - ('(`x`y!(`a;2))', QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), + (b'(`x`y!(`a;2))', QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), [numpy.string_('a'), numpy.int64(2)])), - ('(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)], + (b'(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)], qlist(numpy.array(['first', 'second']), qtype = QSYMBOL_LIST))), - ('(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4'], + (b'(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4'], [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])), - ('`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST), + (b'`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST), [[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), [numpy.string_('a'), numpy.int64(2)]), numpy.float32(5.5)])), - ('flip `abc`def!(1 2 3; 4 5 6)', (qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST), + (b'flip `abc`def!(1 2 3; 4 5 6)', (qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST), [qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([4, 5, 6], dtype=numpy.int64), qtype=QLONG_LIST)], qtype=QTABLE), @@ -238,7 +244,7 @@ qtable(['abc', 'def'], [[1, 2, 3], [4, 5, 6]], **{'abc': QLONG, 'def': QLONG}))), - ('flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)', + (b'flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)', (qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), qlist(numpy.array([98, 42, 126], dtype=numpy.int64), qtype = QLONG_LIST)]), @@ -260,22 +266,22 @@ [['Dent', 'Beeblebrox', 'Prefect'], [98, 42, 126]], **{'name': QSYMBOL, 'iq': QLONG}))), - ('flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")', + (b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")', qtable(qlist(numpy.array(['name', 'iq', 'grade']), qtype = QSYMBOL_LIST), [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), "a c"])), - ('flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))', + (b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))', qtable(qlist(numpy.array(['name', 'iq', 'fullname']), qtype = QSYMBOL_LIST), [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), qlist(numpy.array(["Arthur Dent", "Zaphod Beeblebrox", "Ford Prefect"]), qtype = QSTRING_LIST)])), - ('flip `name`iq`misc!(`Dent`Beeblebrox`Prefect;98 42 126;("The Hitch Hiker\'s Guide to the Galaxy"; 160; 1979.10.12))', + (b'flip `name`iq`misc!(`Dent`Beeblebrox`Prefect;98 42 126;("The Hitch Hiker\'s Guide to the Galaxy"; 160; 1979.10.12))', qtable(qlist(numpy.array(['name', 'iq', 'misc']), qtype = QSYMBOL_LIST), [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), - qlist(numpy.array(["The Hitch Hiker\'s Guide to the Galaxy", 160L, qtemporal(numpy.datetime64('1979-10-12', 'D'), qtype=QDATE)]), qtype = QGENERAL_LIST)])), - ('([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', (qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), + qlist(numpy.array(["The Hitch Hiker\'s Guide to the Galaxy", long(160), qtemporal(numpy.datetime64('1979-10-12', 'D'), qtype=QDATE)]), qtype = QGENERAL_LIST)])), + (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', (qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), [qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST), [qlist(numpy.array([1, 2], dtype=numpy.int64), qtype = QLONG_LIST), qlist(numpy.array([3, 4], dtype=numpy.int64), qtype = QLONG_LIST), @@ -290,12 +296,12 @@ [qlist([1, 2], qtype = QLONG_LIST), qlist([3, 4], qtype = QLONG_LIST), qlist([5, 6, 7], qtype = QLONG_LIST)]]))), - ('([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), + (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), [qlist(numpy.array([1, 2]), qtype = QLONG_LIST), qlist(numpy.array([3, 4]), qtype = QLONG_LIST), qlist(numpy.array([5, 6]), qtype = QLONG_LIST)]])), - ('([] name:`symbol$(); iq:`int$())', (qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), + (b'([] name:`symbol$(); iq:`int$())', (qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), [qlist(numpy.array([], dtype=numpy.string_), qtype = QSYMBOL_LIST), qlist(numpy.array([], dtype=numpy.int32), qtype = QINT_LIST)]), qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), @@ -304,18 +310,15 @@ qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST), [qlist([], qtype = QSYMBOL_LIST), qlist([], qtype = QINT_LIST)]))), - ('([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', + (b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', (qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST), [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)]), - qtable(['pos', 'dates'], - [numpy.array(['d1', 'd2', 'd3']), - numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')]), qtable(['pos', 'dates'], [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')]) )), - ('([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', + (b'([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST), [qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]), qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST), @@ -341,18 +344,21 @@ def init(): def test_writing(): w = qwriter.QWriter(None, 3) - - for query, variants in EXPRESSIONS.iteritems(): + + for query, value in iter(EXPRESSIONS.items()): sys.stdout.write( '%-75s' % query ) - variants = [variants] if not isinstance(variants, tuple) else variants - - for object in variants: + if isinstance(value, tuple): + for object in value: + sys.stdout.write( '.' ) + serialized = binascii.hexlify(w.write(object, 1))[16:].lower() + assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized) + else: sys.stdout.write( '.' ) - serialized = binascii.hexlify(w.write(object, 1))[16:].lower() + serialized = binascii.hexlify(w.write(value, 1))[16:].lower() assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized) - + print('') - + init() From 5f689dc8eb181429866e2a70623bdd27f84c6976 Mon Sep 17 00:00:00 2001 From: Maciej Lach Date: Tue, 4 Aug 2015 14:47:44 +0200 Subject: [PATCH 2/8] Verify Python 3.3 compatibility --- .travis.yml | 1 + CHANGELOG.txt | 2 +- README.rst | 2 +- conda.recipe/meta.yaml | 2 +- qpython/__init__.py | 11 ++++++++++- samples/console.py | 2 ++ setup.py | 2 +- 7 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index ae7c042..c61529f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: python python: - "2.7" + - "3.3" - "3.4" # command to install dependencies diff --git a/CHANGELOG.txt b/CHANGELOG.txt index e461a1f..2236ac5 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,7 +2,7 @@ qPython 1.1.0 [2015.xx.xx] ------------------------------------------------------------------------------ - - Compatibility with Python 2.7 and 3.4 + - Compatibility with Python 2.7, 3.3, and 3.4 ------------------------------------------------------------------------------ qPython 1.0.0 [2015.04.10] diff --git a/README.rst b/README.rst index 0c09ea0..3b8ecec 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,7 @@ qPython is a Python library providing support for interprocess communication bet - Support for kdb+ protocol and types: v3.0, v2.6, v<=2.5 - Uncompression of the IPC data stream - Internal representation of data via numpy arrays (lists, complex types) and numpy data types (atoms) -- Supported on Python 2.7/3.4 and numpy 1.8 +- Supported on Python 2.7/3.3/3.4 and numpy 1.8 For more details please refer to the `documentation`_. diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index f02ef56..382cd0d 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: qpython - version: 1.1.beta + version: "1.1.0b1" build: number: 1 diff --git a/qpython/__init__.py b/qpython/__init__.py index 6d2a0e0..a56c575 100644 --- a/qpython/__init__.py +++ b/qpython/__init__.py @@ -17,7 +17,16 @@ __all__ = ['qconnection', 'qtype', 'qtemporal', 'qcollection'] -__version__ = '1.0.0' +__version__ = '1.1.0b1' + + +try: + from qpython.fastutils import uncompress +except: + __is_cython_enabled__ = False +else: + __is_cython_enabled__ = True + class MetaData(object): '''Utility class for enriching data structures with meta data, e.g. qtype hint.''' diff --git a/samples/console.py b/samples/console.py index 0797827..595fc57 100644 --- a/samples/console.py +++ b/samples/console.py @@ -14,6 +14,7 @@ # limitations under the License. # +import qpython from qpython import qconnection from qpython.qtype import QException @@ -24,6 +25,7 @@ if __name__ == '__main__': + print('qPython %s Cython extensions enabled: %s' % (qpython.__version__, qpython.__is_cython_enabled__)) with qconnection.QConnection(host = 'localhost', port = 5000) as q: print(q) print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())) diff --git a/setup.py b/setup.py index d65e017..4a4487e 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ def read(fname): setup(name = 'qPython', - version = '1.1 beta', + version = '1.1.0b1', description = 'kdb+ interfacing library for Python', long_description=read('README.rst'), From 00bd374dc3320871458ed1a19e7a33882906bfd4 Mon Sep 17 00:00:00 2001 From: Maciej Lach Date: Tue, 4 Aug 2015 15:56:31 +0200 Subject: [PATCH 3/8] Documentation amendments --- doc/source/conf.py | 6 +- doc/source/connection.rst | 4 +- doc/source/pandas.rst | 16 +- doc/source/queries.rst | 27 +-- doc/source/type-conversion.rst | 8 +- doc/source/usage-examples.rst | 298 +++++++++++++++++---------------- samples/console.py | 1 - 7 files changed, 189 insertions(+), 171 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index e3e2674..e3576ba 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -64,16 +64,16 @@ def __getattr__(cls, name): # General information about the project. project = u'qPython' -copyright = u'2014, DEVnet' +copyright = u'2014-2015, DEVnet' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '1.0' +version = '1.1' # The full version, including alpha/beta/rc tags. -release = '1.0-beta' +release = '1.1.0b1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/connection.rst b/doc/source/connection.rst index 9f63417..caeeacc 100644 --- a/doc/source/connection.rst +++ b/doc/source/connection.rst @@ -26,8 +26,8 @@ can be used with a ``with`` statement: :: with qconnection.QConnection(host = 'localhost', port = 5000) as q: - print q - print q('{`int$ til x}', 10) + print(q) + print(q('{`int$ til x}', 10)) Q parser configuration diff --git a/doc/source/pandas.rst b/doc/source/pandas.rst index 6aac61a..7387366 100644 --- a/doc/source/pandas.rst +++ b/doc/source/pandas.rst @@ -17,35 +17,35 @@ For example: >>> with qconnection.QConnection(host = 'localhost', port = 5000, pandas = True) as q: >>> ds = q('(1i;0Ni;3i)', pandas = True) - >>> print ds + >>> print(ds) 0 1 1 NaN 2 3 dtype: float64 - >>> print ds.meta + >>> print(ds.meta) metadata(qtype=6) >>> df = q('flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))') - >>> print df + >>> print(df) name iq fullname 0 Dent 98 Arthur Dent 1 Beeblebrox 42 Zaphod Beeblebrox 2 Prefect 126 Ford Prefect - >>> print df.meta + >>> print(df.meta) metadata(iq=7, fullname=0, qtype=98, name=11) - >>> print q('type', df) + >>> print(q('type', df)) 98 >>> df = q('([eid:1001 0N 1003;sym:`foo`bar`] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))') - >>> print df + >>> print(df) pos dates eid sym 1001 foo d1 2001-01-01 NaN bar d2 2000-05-01 1003 d3 NaT - >>> print df.meta + >>> print(df.meta) metadata(dates=14, qtype=99, eid=7, sym=11, pos=11) - >>> print q('type', df) + >>> print(q('type', df)) 99 diff --git a/doc/source/queries.rst b/doc/source/queries.rst index f4d0b9b..160ced3 100644 --- a/doc/source/queries.rst +++ b/doc/source/queries.rst @@ -35,26 +35,26 @@ Synchronous queries Executes a q expression: - >>> print q.sync('til 10') + >>> print(q.sync('til 10')) [0 1 2 3 4 5 6 7 8 9] Executes an anonymous q function with a single parameter: - >>> print q.sync('{til x}', 10) + >>> print(q.sync('{til x}', 10)) [0 1 2 3 4 5 6 7 8 9] Executes an anonymous q function with two parameters: - >>> print q.sync('{y + til x}', 10, 1) + >>> print(q.sync('{y + til x}', 10, 1)) [ 1 2 3 4 5 6 7 8 9 10] - >>> print q.sync('{y + til x}', *[10, 1]) + >>> print(q.sync('{y + til x}', *[10, 1])) [ 1 2 3 4 5 6 7 8 9 10] The :class:`.QConnection` class implements the :func:`~qpython.qconnection.QConnection.__call__` method. This allows :class:`.QConnection` instance to be called as a function: - >>> print q('{y + til x}', 10, 1) + >>> print(q('{y + til x}', 10, 1)) [ 1 2 3 4 5 6 7 8 9 10] @@ -84,25 +84,25 @@ For example: - Retrieves query result along with meta-information: >>> q.query(qconnection.MessageType.SYNC,'{x}', 10) ->>> print q.receive(data_only = False, raw = False) +>>> print(q.receive(data_only = False, raw = False)) QMessage: message type: 2, data size: 13, is_compressed: False, data: 10 - Retrieves parsed query result: >>> q.query(qconnection.MessageType.SYNC,'{x}', 10) ->>> print q.receive(data_only = True, raw = False) +>>> print(q.receive(data_only = True, raw = False)) 10 ->>> q.sync('asynchMult:{[a;b] res:a*b; (neg .z.w)(res) }'); +>>> q.sync('asynchMult:{[a;b] res:a*b; (neg .z.w)(res) }') >>> q.async('asynchMult', 2, 3) ->>> print q.receive() +>>> print(q.receive()) 6 - Retrieves not-parsed (raw) query result: >>> from binascii import hexlify >>> q.query(qconnection.MessageType.SYNC,'{x}', 10) ->>> print hexlify(q.receive(data_only = True, raw = True)) +>>> print(hexlify(q.receive(data_only = True, raw = True))) fa0a000000 @@ -119,17 +119,18 @@ Both methods accepts the `options` keywords arguments:: >>> query = "{[x] 0Nd, `date$til x}" >>> # retrieve function call as raw byte buffer - >>> print binascii.hexlify(q(query, 5, raw = True)) + >>> from binascii import hexlify + >>> print(binascii.hexlify(q(query, 5, raw = True))) 0e0006000000000000800000000001000000020000000300000004000000 >>> # perform a synchronous call and parse dates vector to numpy array - >>> print q.sync(query, 5, numpy_temporals = True) + >>> print(q.sync(query, 5, numpy_temporals = True)) ['NaT' '2000-01-01' '2000-01-02' '2000-01-03' '2000-01-04' '2000-01-05'] >>> # perform a synchronous call >>> q.query(qconnection.MessageType.SYNC, query, 3) >>> # retrieve query result and represent dates vector as raw data wrapped in QTemporalList - >>> print q.receive(numpy_temporals = False) + >>> print(q.receive(numpy_temporals = False)) [NaT [metadata(qtype=-14)] 2000-01-01 [metadata(qtype=-14)] 2000-01-02 [metadata(qtype=-14)] 2000-01-03 [metadata(qtype=-14)]] diff --git a/doc/source/type-conversion.rst b/doc/source/type-conversion.rst index 6db59b2..fcbd7e4 100644 --- a/doc/source/type-conversion.rst +++ b/doc/source/type-conversion.rst @@ -183,12 +183,12 @@ for ``month``\s etc.) and provides accessors which allow to convert raw data to :: >>> v = q.sync("2001.01.01 2000.05.01 0Nd", numpy_temporals = False) - >>> print '%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v) + >>> print('%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v)) dtype: int32 qtype: -14: [2001-01-01 [metadata(qtype=-14)] 2000-05-01 [metadata(qtype=-14)] NaT [metadata(qtype=-14)]] >>> v = q.sync("2000.01.04D05:36:57.600 0Np", numpy_temporals = False) - >>> print '%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v) + >>> print('%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v)) dtype: int64 qtype: -12: [2000-01-04T05:36:57.600000000+0100 [metadata(qtype=-12)] NaT [metadata(qtype=-12)]] @@ -203,11 +203,11 @@ via :class:`~.qconnection.QConnection` constructor or as parameter to functions: :: >>> v = q.sync("2001.01.01 2000.05.01 0Nd", numpy_temporals = True) - >>> print '%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v) + >>> print('%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v)) dtype: datetime64[D] qtype: -14: ['2001-01-01' '2000-05-01' 'NaT'] >>> v = q.sync("2000.01.04D05:36:57.600 0Np", numpy_temporals = True) - >>> print '%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v) + >>> print('%s dtype: %s qtype: %d: %s' % (type(v), v.dtype, v.meta.qtype, v)) dtype: datetime64[ns] qtype: -12: ['2000-01-04T05:36:57.600000000+0100' 'NaT'] diff --git a/doc/source/usage-examples.rst b/doc/source/usage-examples.rst index aa27e16..b82c01e 100644 --- a/doc/source/usage-examples.rst +++ b/doc/source/usage-examples.rst @@ -10,29 +10,30 @@ Following example presents how to execute simple, synchronous query against a re from qpython import qconnection + if __name__ == '__main__': # create connection object - q = qconnection.QConnection(host = 'localhost', port = 5000) + q = qconnection.QConnection(host='localhost', port=5000) # initialize connection q.open() - print q - print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()) + print(q) + print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())) # simple query execution via: QConnection.__call__ data = q('{`int$ til x}', 10) - print 'type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data) - + print('type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data)) + # simple query execution via: QConnection.sync data = q.sync('{`long$ til x}', 10) - print 'type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data) - + print('type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data)) + # low-level query and read - q.query(qconnection.MessageType.SYNC,'{`short$ til x}', 10) # sends a SYNC query - msg = q.receive(data_only = False, raw = False) # retrieve entire message - print 'type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(msg), msg.type, msg.size, msg.is_compressed) + q.query(qconnection.MessageType.SYNC, '{`short$ til x}', 10) # sends a SYNC query + msg = q.receive(data_only=False, raw=False) # retrieve entire message + print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(msg), msg.type, msg.size, msg.is_compressed)) data = msg.data - print 'type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data) + print('type: %s, numpy.dtype: %s, meta.qtype: %s, data: %s ' % (type(data), data.dtype, data.meta.qtype, data)) # close connection q.close() @@ -69,33 +70,33 @@ Following example presents how to execute simple, asynchronous query against a r def __init__(self, q): super(ListenerThread, self).__init__() self.q = q - self._stop = threading.Event() + self._stopper = threading.Event() def stop(self): - self._stop.set() + self._stopper.set() def stopped(self): - return self._stop.isSet() + return self._stopper.isSet() def run(self): while not self.stopped(): - print '.' + print('.') try: message = self.q.receive(data_only = False, raw = False) # retrieve entire message if message.type != MessageType.ASYNC: - print 'Unexpected message, expected message of type: ASYNC' + print('Unexpected message, expected message of type: ASYNC') - print 'type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed) - print message.data + print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed)) + print(message.data) if isinstance(message.data, QDictionary): # stop after 10th query - if message.data['queryid'] == 9: + if message.data[b'queryid'] == 9: self.stop() - except QException, e: - print e + except QException as e: + print(e) if __name__ == '__main__': @@ -104,8 +105,8 @@ Following example presents how to execute simple, asynchronous query against a r # initialize connection q.open() - print q - print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()) + print(q) + print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())) try: # definition of asynchronous multiply function @@ -117,10 +118,10 @@ Following example presents how to execute simple, asynchronous query against a r t = ListenerThread(q) t.start() - for x in xrange(10): + for x in range(10): a = random.randint(1, 100) b = random.randint(1, 100) - print 'Asynchronous call with queryid=%s with arguments: %s, %s' % (x, a, b) + print('Asynchronous call with queryid=%s with arguments: %s, %s' % (x, a, b)) q.async('asynchMult', x, a, b); time.sleep(1) @@ -135,31 +136,38 @@ This example depicts how to create a simple interactive console for communicatio .. code:: python + import qpython from qpython import qconnection from qpython.qtype import QException + try: + input = raw_input + except NameError: + pass + if __name__ == '__main__': + print('qPython %s Cython extensions enabled: %s' % (qpython.__version__, qpython.__is_cython_enabled__)) with qconnection.QConnection(host = 'localhost', port = 5000) as q: - print q - print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()) - + print(q) + print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())) + while True: try: - x = raw_input('Q)') + x = input('Q)') except EOFError: - print + print('') break - + if x == '\\\\': break - + try: result = q(x) - print type(result) - print result - except QException, msg: - print 'q error: \'%s' % msg + print(type(result)) + print(result) + except QException as msg: + print('q error: \'%s' % msg) Twisted integration @@ -182,7 +190,10 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui from qpython.qreader import QReader from qpython.qwriter import QWriter, QWriterException + + class IPCProtocol(Protocol): + class State(object): UNKNOWN = -1 HANDSHAKE = 0 @@ -193,37 +204,37 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui self.credentials = self.factory.username + ':' + self.factory.password if self.factory.password else '' self.transport.write(self.credentials + '\3\0') - + self._message = None def dataReceived(self, data): if self.state == IPCProtocol.State.CONNECTED: try: if not self._message: - self._message = self._reader.read_header(source = data) + self._message = self._reader.read_header(source=data) self._buffer = '' - + self._buffer += data buffer_len = len(self._buffer) if self._buffer else 0 - + while self._message and self._message.size <= buffer_len: complete_message = self._buffer[:self._message.size] - + if buffer_len > self._message.size: self._buffer = self._buffer[self._message.size:] buffer_len = len(self._buffer) if self._buffer else 0 - self._message = self._reader.read_header(source = self._buffer) + self._message = self._reader.read_header(source=self._buffer) else: self._message = None self._buffer = '' buffer_len = 0 - self.factory.onMessage(self._reader.read(source = complete_message)) + self.factory.onMessage(self._reader.read(source=complete_message, numpy_temporals=True)) except: self.factory.onError(sys.exc_info()) self._message = None self._buffer = '' - + elif self.state == IPCProtocol.State.UNKNOWN: # handshake if len(data) == 1: @@ -231,7 +242,7 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui else: self.state = IPCProtocol.State.HANDSHAKE self.transport.write(self.credentials + '\0') - + else: # protocol version fallback if len(data) == 1: @@ -242,8 +253,8 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui def _init(self, data): self.state = IPCProtocol.State.CONNECTED self.protocol_version = min(struct.unpack('B', data)[0], 3) - self._writer = QWriter(stream = None, protocol_version = self.protocol_version) - self._reader = QReader(stream = None) + self._writer = QWriter(stream=None, protocol_version=self.protocol_version) + self._reader = QReader(stream=None) self.factory.clientReady(self) @@ -257,7 +268,9 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui self.transport.write(self._writer.write([query] + list(parameters), msg_type)) + class IPCClientFactory(ClientFactory): + protocol = IPCProtocol def __init__(self, username, password, connect_success_callback, connect_fail_callback, data_callback, error_callback): @@ -271,8 +284,9 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui self.data_callback = data_callback self.error_callback = error_callback + def clientConnectionLost(self, connector, reason): - print 'Lost connection. Reason:', reason + print('Lost connection. Reason: %s' % reason) # connector.connect() def clientConnectionFailed(self, connector, reason): @@ -297,20 +311,25 @@ This example presents how the `qPython` can be used along with `Twisted`_ to bui self.client.query(msg_type, query, *parameters) + def onConnectSuccess(source): - print 'Connected, protocol version: ', source.client.protocol_version - source.query(MessageType.SYNC, '.z.ts:{(handle)((1000*(1 ? 100))[0] ? 100)}') + print('Connected, protocol version: %s' % source.client.protocol_version) + source.query(MessageType.SYNC, '.z.ts:{(handle)(`timestamp$100?1000000000000000000)}') source.query(MessageType.SYNC, '.u.sub:{[t;s] handle:: neg .z.w}') source.query(MessageType.ASYNC, '.u.sub', 'trade', '') + def onConnectFail(source, reason): - print 'Connection refused: ', reason + print('Connection refused: %s' % reason) + def onMessage(source, message): - print 'Received: ', message.type, message.data + print('Received: %s %s' % (message.type, message.data)) + def onError(source, error): - print 'Error: ', error + print('Error: %s' % error) + if __name__ == '__main__': factory = IPCClientFactory('user', 'pwd', onConnectSuccess, onConnectFail, onMessage, onError) @@ -325,69 +344,68 @@ This example depicts how to subscribe to standard kdb+ tickerplant service: .. code:: python - import numpy - import threading - import sys - - from qpython import qconnection - from qpython.qtype import QException - from qpython.qconnection import MessageType - from qpython.qcollection import QTable - - - class ListenerThread(threading.Thread): - - def __init__(self, q): - super(ListenerThread, self).__init__() - self.q = q - self._stop = threading.Event() - - def stop(self): - self._stop.set() - - def stopped(self): - return self._stop.isSet() - - def run(self): - while not self.stopped(): - print '.' - try: - message = self.q.receive(data_only = False, raw = False) # retrieve entire message - - if message.type != MessageType.ASYNC: - print 'Unexpected message, expected message of type: ASYNC' - - print 'type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed) - - if isinstance(message.data, list): - # unpack upd message - if len(message.data) == 3 and message.data[0] == 'upd' and isinstance(message.data[2], QTable): - for row in message.data[2]: - print row - - except QException, e: - print e - - - if __name__ == '__main__': - with qconnection.QConnection(host = 'localhost', port = 17010) as q: - print q - print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()) - print 'Press to close application' - - # subscribe to tick - response = q.sync('.u.sub', numpy.string_('trade'), numpy.string_('')) - # get table model - if isinstance(response[1], QTable): - print '%s table data model: %s' % (response[0], response[1].dtype) - - t = ListenerThread(q) - t.start() - - sys.stdin.readline() - - t.stop() - + import numpy + import threading + import sys + + from qpython import qconnection + from qpython.qtype import QException + from qpython.qconnection import MessageType + from qpython.qcollection import QTable + + + class ListenerThread(threading.Thread): + + def __init__(self, q): + super(ListenerThread, self).__init__() + self.q = q + self._stopper = threading.Event() + + def stopit(self): + self._stopper.set() + + def stopped(self): + return self._stopper.is_set() + + def run(self): + while not self.stopped(): + print('.') + try: + message = self.q.receive(data_only = False, raw = False) # retrieve entire message + + if message.type != MessageType.ASYNC: + print('Unexpected message, expected message of type: ASYNC') + + print('type: %s, message type: %s, data size: %s, is_compressed: %s ' % (type(message), message.type, message.size, message.is_compressed)) + + if isinstance(message.data, list): + # unpack upd message + if len(message.data) == 3 and message.data[0] == b'upd' and isinstance(message.data[2], QTable): + for row in message.data[2]: + print(row) + + except QException as e: + print(e) + + + if __name__ == '__main__': + with qconnection.QConnection(host = 'localhost', port = 17010) as q: + print(q) + print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())) + print('Press to close application') + + # subscribe to tick + response = q.sync('.u.sub', numpy.string_('trade'), numpy.string_('')) + # get table model + if isinstance(response[1], QTable): + print('%s table data model: %s' % (response[0], response[1].dtype)) + + t = ListenerThread(q) + t.start() + + sys.stdin.readline() + + t.stopit() Data publisher @@ -410,59 +428,59 @@ This example shows how to stream data to the kdb+ process using standard tickerp class PublisherThread(threading.Thread): - + def __init__(self, q): super(PublisherThread, self).__init__() self.q = q - self._stop = threading.Event() + self._stopper = threading.Event() def stop(self): - self._stop.set() + self._stopper.set() def stopped(self): - return self._stop.isSet() + return self._stopper.isSet() def run(self): while not self.stopped(): - print '.' + print('.') try: # publish data to tick # function: .u.upd # table: ask self.q.sync('.u.upd', numpy.string_('ask'), self.get_ask_data()) - + time.sleep(1) - except QException, e: - print e + except QException as e: + print(e) except: self.stop() - + def get_ask_data(self): c = random.randint(1, 10) - + today = numpy.datetime64(datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)) - - time = [numpy.timedelta64((numpy.datetime64(datetime.datetime.now()) - today), 'ms') for x in xrange(c)] - instr = ['instr_%d' % random.randint(1, 100) for x in xrange(c)] - src = ['qPython' for x in xrange(c)] - ask = [random.random() * random.randint(1, 100) for x in xrange(c)] - + + time = [numpy.timedelta64((numpy.datetime64(datetime.datetime.now()) - today), 'ms') for x in range(c)] + instr = ['instr_%d' % random.randint(1, 100) for x in range(c)] + src = ['qPython' for x in range(c)] + ask = [random.random() * random.randint(1, 100) for x in range(c)] + data = [qlist(time, qtype=QTIME_LIST), qlist(instr, qtype=QSYMBOL_LIST), qlist(src, qtype=QSYMBOL_LIST), qlist(ask, qtype=QFLOAT_LIST)] - print data + print(data) return data - - + + if __name__ == '__main__': with qconnection.QConnection(host='localhost', port=17010) as q: - print q - print 'IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()) - print 'Press to close application' + print(q) + print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected())) + print('Press to close application') t = PublisherThread(q) t.start() - + sys.stdin.readline() - + t.stop() t.join() diff --git a/samples/console.py b/samples/console.py index 595fc57..e44f6ad 100644 --- a/samples/console.py +++ b/samples/console.py @@ -46,4 +46,3 @@ print(result) except QException as msg: print('q error: \'%s' % msg) - From ea12fe9a2a6f19ddda06b468be9b212d49106bd6 Mon Sep 17 00:00:00 2001 From: Maciej Lach Date: Wed, 5 Aug 2015 09:56:35 +0200 Subject: [PATCH 4/8] Travis: Migrating from legacy to container-based infrastructure --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index c61529f..fce76dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,7 @@ language: python + +sudo: false + python: - "2.7" - "3.3" From 19cc5d5d439068093a395cfab7bc13aec7bc1b7e Mon Sep 17 00:00:00 2001 From: Maciej Lach Date: Thu, 3 Sep 2015 16:25:57 +0200 Subject: [PATCH 5/8] Attempt to fix readthedocs build --- setup.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 122f2cc..6581fd9 100644 --- a/setup.py +++ b/setup.py @@ -17,9 +17,14 @@ from distutils.core import setup from qpython import __version__ -import numpy import os +try: + import numpy + include_dirs = [numpy.get_include()] +except: + include_dirs = [] + try: from Cython.Build import cythonize except ImportError: @@ -49,7 +54,7 @@ def read(fname): license = 'Apache License Version 2.0', ext_modules = ext_modules, - include_dirs = [numpy.get_include()], + include_dirs = include_dirs, keywords = ['kdb+', 'q'], classifiers=[ From ac0e97f5f3002f31acd57072516d437149ce4903 Mon Sep 17 00:00:00 2001 From: Maciej Lach Date: Fri, 4 Sep 2015 14:37:57 +0200 Subject: [PATCH 6/8] Add classifiers for py 3.3/3.4/2.7 --- setup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup.py b/setup.py index 6581fd9..1381b40 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,11 @@ def read(fname): 'Operating System :: POSIX', 'Operating System :: Unix', 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', 'Topic :: Database :: Front-Ends', 'Topic :: Scientific/Engineering', 'Topic :: Software Development', From 7cc6f49734e2853b3725e8db7a74aad2e928ed34 Mon Sep 17 00:00:00 2001 From: Maciej Lach Date: Fri, 4 Sep 2015 14:41:52 +0200 Subject: [PATCH 7/8] Remove duplicated test case --- tests/qwriter_test.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/qwriter_test.py b/tests/qwriter_test.py index 9e84503..ff195a8 100644 --- a/tests/qwriter_test.py +++ b/tests/qwriter_test.py @@ -28,11 +28,6 @@ BINARY = OrderedDict() EXPRESSIONS = OrderedDict(( - (b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")', - qtable(qlist(numpy.array(['name', 'iq', 'grade']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), - qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), - "a c"])), (b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)', qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST)), (b'"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')), From 48bba4a6672f11823569bfd69b1d16113a0a6375 Mon Sep 17 00:00:00 2001 From: Maciej Lach Date: Wed, 25 Nov 2015 12:36:17 +0100 Subject: [PATCH 8/8] Version and changelog update --- CHANGELOG.txt | 2 +- conda.recipe/meta.yaml | 2 +- qpython/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index b482639..9dc7877 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,5 +1,5 @@ ------------------------------------------------------------------------------ - qPython 1.1.0 [2015.xx.xx] + qPython 1.1.0 [2015.11.25] ------------------------------------------------------------------------------ - Compatibility with Python 2.7, 3.3, and 3.4 diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 52deb7b..adf1866 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: qpython - version: "1.1.0b3" + version: "1.1.0" build: number: 1 diff --git a/qpython/__init__.py b/qpython/__init__.py index f945548..3abaebc 100644 --- a/qpython/__init__.py +++ b/qpython/__init__.py @@ -17,7 +17,7 @@ __all__ = ['qconnection', 'qtype', 'qtemporal', 'qcollection'] -__version__ = '1.1.0b3' +__version__ = '1.1.0'