From e4af4b5c64a3f31d9a045259e79fb70bbfa4f9aa Mon Sep 17 00:00:00 2001 From: Daniel Nugent Date: Thu, 30 Aug 2018 19:24:34 +0000 Subject: [PATCH 1/3] Cleaned up errors and warnings --- .gitignore | 1 + qpython/_pandas.py | 12 ++++++------ qpython/qreader.py | 4 ++-- qpython/qtemporal.py | 19 ++++++++++--------- qpython/qwriter.py | 5 ++++- tests/qreader_test.py | 10 +++++++++- tests/qtypes_test.py | 27 +++++++++++++++------------ 7 files changed, 47 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index 4955aad..9f45451 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,7 @@ dist # coverage .coverage __conda_version__.txt +.pytest_cache # OS generated files # ###################### diff --git a/qpython/_pandas.py b/qpython/_pandas.py index 7686125..adc9b54 100644 --- a/qpython/_pandas.py +++ b/qpython/_pandas.py @@ -58,8 +58,8 @@ def _read_dictionary(self, qtype = QDICTIONARY): return table else: - keys = keys if not isinstance(keys, pandas.Series) else keys.as_matrix() - values = values if not isinstance(values, pandas.Series) else values.as_matrix() + keys = keys if not isinstance(keys, pandas.Series) else keys.values + values = values if not isinstance(values, pandas.Series) else values.values return QDictionary(keys, values) else: return QReader._read_dictionary(self, qtype = qtype) @@ -168,19 +168,19 @@ def _write_pandas_series(self, data, qtype = None): raise QWriterException('Unable to serialize pandas series %s' % data) if qtype == QGENERAL_LIST: - self._write_generic_list(data.as_matrix()) + self._write_generic_list(data.values) elif qtype == QCHAR: - self._write_string(data.replace(numpy.nan, ' ').as_matrix().astype(numpy.string_).tostring()) + self._write_string(data.replace(numpy.nan, ' ').values.astype(numpy.string_).tostring()) elif data.dtype.type not in (numpy.datetime64, numpy.timedelta64): data = data.fillna(QNULLMAP[-abs(qtype)][1]) - data = data.as_matrix() + data = data.values if PY_TYPE[qtype] != data.dtype: data = data.astype(PY_TYPE[qtype]) self._write_list(data, qtype = qtype) else: - data = data.as_matrix() + data = data.values data = data.astype(TEMPORAL_Q_TYPE[qtype]) self._write_list(data, qtype = qtype) diff --git a/qpython/qreader.py b/qpython/qreader.py index 3a602e3..2dbdf54 100644 --- a/qpython/qreader.py +++ b/qpython/qreader.py @@ -200,7 +200,7 @@ def read_data(self, message_size, is_compressed = False, **options): uncompressed_size = -8 + self._buffer.get_int() compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12) - raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8) + raw_data = numpy.frombuffer(compressed_data, dtype = numpy.uint8) if uncompressed_size <= 0: raise QReaderException('Error while data decompression.') @@ -296,7 +296,7 @@ def _read_list(self, qtype): return qlist(data, qtype = qtype, adjust_dtype = False) elif conversion: raw = self._buffer.raw(length * ATOM_SIZE[qtype]) - data = numpy.fromstring(raw, dtype = conversion) + data = numpy.frombuffer(raw, dtype = conversion) if not self._is_native: data.byteswap(True) diff --git a/qpython/qtemporal.py b/qpython/qtemporal.py index bb9042c..ffd6957 100644 --- a/qpython/qtemporal.py +++ b/qpython/qtemporal.py @@ -72,7 +72,8 @@ def __str__(self): def __eq__(self, other): return (isinstance(other, self.__class__) and self.meta.qtype == other.meta.qtype - and self._datetime == other._datetime) + and (numpy.isnat(self._datetime) and numpy.isnat(other._datetime)) + or self._datetime == other._datetime) def __ne__(self, other): return not self.__eq__(other) @@ -231,7 +232,7 @@ def _to_qmonth(dt): if t_dt == numpy.int32: return dt elif t_dt == numpy.datetime64: - return (dt - _EPOCH_QMONTH).astype(int) if not dt == _NUMPY_NULL[QMONTH] else _QMONTH_NULL + return (dt - _EPOCH_QMONTH).astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QMONTH]) else _QMONTH_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -250,7 +251,7 @@ def _to_qdate(dt): if t_dt == numpy.int32: return dt elif t_dt == numpy.datetime64: - return (dt - _EPOCH_QDATE).astype(int) if not dt == _NUMPY_NULL[QDATE] else _QDATE_NULL + return (dt - _EPOCH_QDATE).astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QDATE]) else _QDATE_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -269,7 +270,7 @@ def _to_qdatetime(dt): if t_dt == numpy.float64: return dt elif t_dt == numpy.datetime64: - return (dt - _EPOCH_QDATETIME).astype(float) / _MILLIS_PER_DAY if not dt == _NUMPY_NULL[QDATETIME] else _QDATETIME_NULL + return (dt - _EPOCH_QDATETIME).astype(float) / _MILLIS_PER_DAY if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QDATETIME]) else _QDATETIME_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -288,7 +289,7 @@ def _to_qminute(dt): if t_dt == numpy.int32: return dt elif t_dt == numpy.timedelta64: - return dt.astype(int) if not dt == _NUMPY_NULL[QMINUTE] else _QMINUTE_NULL + return dt.astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QMINUTE]) else _QMINUTE_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -307,7 +308,7 @@ def _to_qsecond(dt): if t_dt == numpy.int32: return dt elif t_dt == numpy.timedelta64: - return dt.astype(int) if not dt == _NUMPY_NULL[QSECOND] else _QSECOND_NULL + return dt.astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QSECOND]) else _QSECOND_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -326,7 +327,7 @@ def _to_qtime(dt): if t_dt == numpy.int32: return dt elif t_dt == numpy.timedelta64: - return dt.astype(int) if not dt == _NUMPY_NULL[QTIME] else _QTIME_NULL + return dt.astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QTIME]) else _QTIME_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -345,7 +346,7 @@ def _to_qtimestamp(dt): if t_dt == numpy.int64: return dt elif t_dt == numpy.datetime64: - return (dt - _EPOCH_TIMESTAMP).astype(longlong) if not dt == _NUMPY_NULL[QTIMESTAMP] else _QTIMESTAMP_NULL + return (dt - _EPOCH_TIMESTAMP).astype(longlong) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QTIMESTAMP]) else _QTIMESTAMP_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -364,7 +365,7 @@ def _to_qtimespan(dt): if t_dt == numpy.int64: return dt elif t_dt == numpy.timedelta64: - return dt.astype(longlong) if not dt == _NUMPY_NULL[QTIMESPAN] else _QTIMESTAMP_NULL + return dt.astype(longlong) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QTIMESPAN]) else _QTIMESTAMP_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) diff --git a/qpython/qwriter.py b/qpython/qwriter.py index d24fa5d..5bda6f8 100644 --- a/qpython/qwriter.py +++ b/qpython/qwriter.py @@ -145,7 +145,10 @@ def _write_atom(self, data, qtype): try: self._buffer.write(struct.pack('b', qtype)) fmt = STRUCT_MAP[qtype] - self._buffer.write(struct.pack(fmt, data)) + if isinstance(data,numpy.bool_): + self._buffer.write(struct.pack(fmt,bool(data))) + else: + self._buffer.write(struct.pack(fmt, data)) except KeyError: raise QWriterException('Unable to serialize type: %s' % data.__class__ if isinstance(data, object) else type(data)) diff --git a/tests/qreader_test.py b/tests/qreader_test.py index c44363d..19569a4 100644 --- a/tests/qreader_test.py +++ b/tests/qreader_test.py @@ -28,7 +28,7 @@ from collections import OrderedDict from qpython import qreader from qpython.qtype import * # @UnusedWildImport -from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, qtable, QKeyedTable +from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, qtable, QKeyedTable, QTable from qpython.qtemporal import qtemporal, QTemporal @@ -278,12 +278,20 @@ def arrays_equal(left, right): def compare(left, right): if type(left) in [float, numpy.float32, numpy.float64] and numpy.isnan(left): return numpy.isnan(right) + if type(left) in [numpy.datetime64, numpy.timedelta64] and numpy.isnat(left): + return numpy.isnat(right) if type(left) == QTemporal and isinstance(left.raw, float) and numpy.isnan(left.raw): return numpy.isnan(right.raw) + elif type(left) == QTemporal and isinstance(left.raw, numpy.datetime64) and numpy.isnat(left.raw): + return numpy.isnat(right.raw) + elif type(left) == QTemporal and isinstance(left.raw, numpy.timedelta64) and numpy.isnat(left.raw): + return numpy.isnat(right.raw) elif type(left) in [list, tuple, numpy.ndarray, QList, QTemporalList]: return arrays_equal(left, right) elif type(left) == QFunction: return type(right) == QFunction + elif type(left) == QTable: + return left.dtype == right.dtype and all(arrays_equal(left[n],right[n]) for n in left.dtype.names) else: return left == right diff --git a/tests/qtypes_test.py b/tests/qtypes_test.py index bb1ce92..407c0c6 100644 --- a/tests/qtypes_test.py +++ b/tests/qtypes_test.py @@ -195,7 +195,7 @@ def test_array_to_raw_qtemporal(): assert na[x] == x - 365 x += 1 - na_dt = numpy.arange('1999-01-01T00:00:00.000Z', '2001-01-04T05:36:57.600Z', 12345678, dtype='datetime64[ms]') + na_dt = numpy.arange('1999-01-01T00:00:00.000', '2001-01-04T05:36:57.600', 12345678, dtype='datetime64[ms]') na = array_to_raw_qtemporal(na_dt, qtype=QDATETIME_LIST) assert na.dtype == numpy.float64 @@ -208,7 +208,7 @@ def test_array_to_raw_qtemporal(): ref = (x * step) - 365 assert abs(na[x] - ref) < 0.1, '%s %s' %(na[x], ref) - na_dt = numpy.arange('1999-01-01T00:00:00.000Z', '2001-01-04T05:36:57.600Z', 1234567890000, dtype='datetime64[ns]') + na_dt = numpy.arange('1999-01-01T00:00:00.000', '2001-01-04T05:36:57.600', 1234567890000, dtype='datetime64[ns]') na = array_to_raw_qtemporal(na_dt, qtype=QTIMESTAMP_LIST) assert na.dtype == numpy.int64 @@ -249,7 +249,7 @@ def test_array_from_raw_qtemporal(): assert str(na_dt.dtype).startswith('datetime64[M]') for x in range(len(na_dt)): - if na_dt[x] != numpy.datetime64('NaT', 'M'): + if not numpy.isnat(na_dt[x]): assert na_dt[x].astype(int) == raw[x] + 360 else: assert raw[x] == qnull(QMONTH) @@ -259,7 +259,7 @@ def test_array_from_raw_qtemporal(): assert str(na_dt.dtype).startswith('datetime64[D]') for x in range(len(na_dt)): - if na_dt[x] != numpy.datetime64('NaT', 'D'): + if not numpy.isnat(na_dt[x]): assert na_dt[x].astype(int) == raw[x] + 10957 else: assert raw[x] == qnull(QDATE) @@ -269,7 +269,7 @@ def test_array_from_raw_qtemporal(): assert str(na_dt.dtype).startswith('timedelta64[m]') for x in range(len(na_dt)): - if na_dt[x] != numpy.timedelta64('NaT', 'm'): + if not numpy.isnat(na_dt[x]): assert na_dt[x].astype(int) == raw[x] else: assert raw[x] == qnull(QMINUTE) @@ -279,7 +279,7 @@ def test_array_from_raw_qtemporal(): assert str(na_dt.dtype).startswith('timedelta64[s]') for x in range(len(na_dt)): - if na_dt[x] != numpy.timedelta64('NaT', 's'): + if not numpy.isnat(na_dt[x]): assert na_dt[x].astype(int) == raw[x] else: assert raw[x] == qnull(QSECOND) @@ -289,7 +289,7 @@ def test_array_from_raw_qtemporal(): assert str(na_dt.dtype).startswith('timedelta64[ms]') for x in range(len(na_dt)): - if na_dt[x] != numpy.timedelta64('NaT', 'ms'): + if not numpy.isnat(na_dt[x]): assert na_dt[x].astype(int) == raw[x] else: assert raw[x] == qnull(QTIME) @@ -299,7 +299,7 @@ def test_array_from_raw_qtemporal(): assert str(na_dt.dtype).startswith('timedelta64[ns]') for x in range(len(na_dt)): - if na_dt[x] != numpy.timedelta64('NaT', 'ns'): + if not numpy.isnat(na_dt[x]): assert na_dt[x].astype(numpy.int64) == raw[x] else: assert raw[x] == qnull(QTIMESPAN) @@ -309,19 +309,22 @@ def test_array_from_raw_qtemporal(): assert str(na_dt.dtype).startswith('datetime64[ns]') for x in range(len(na_dt)): - if na_dt[x] != numpy.datetime64('NaT', 'ns'): - assert na_dt[x].astype(numpy.int64) == raw[x] + numpy.datetime64('2000-01-01T00:00:00Z', 'ns').astype(numpy.int64) + if not numpy.isnat(na_dt[x]): + assert na_dt[x].astype(numpy.int64) == raw[x] + numpy.datetime64('2000-01-01T00:00:00', 'ns').astype(numpy.int64) else: assert raw[x] == qnull(QTIMESTAMP) raw = numpy.array([3.234, qnull(QDATETIME)]) na_dt = array_from_raw_qtemporal(raw, qtype=QDATETIME) - ref = numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]) + ref = numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')]) assert str(na_dt.dtype).startswith('datetime64[ms]') for x in range(len(na_dt)): - assert na_dt[x] == ref[x] + if not numpy.isnat(na_dt[x]): + assert na_dt[x] == ref[x] + else: + assert numpy.isnan(raw[x]) test_is_null() From 80d106714cd2d6d907e9f782848a90ec2442b6d1 Mon Sep 17 00:00:00 2001 From: Daniel Nugent Date: Fri, 31 Aug 2018 08:16:43 +0000 Subject: [PATCH 2/3] Renamed 'async' to 'send'. Bump to v2 --- CHANGELOG.txt | 8 ++++++++ doc/source/connection.rst | 2 +- doc/source/queries.rst | 12 ++++++------ doc/source/usage-examples.rst | 2 +- qpython/qconnection.py | 6 +++--- samples/async_query.py | 2 +- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 983b9dc..08a164b 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,11 @@ +------------------------------------------------------------------------------ + qPython 2.0.0 [2018.08.31] +------------------------------------------------------------------------------ + + - BREAKING CHANGE: Renamed QConnection.async to QConnection.send to + support Python 3.7 because of async keyword promotion. + - Fix DeprecationWarnings and FutureWarnings + ------------------------------------------------------------------------------ qPython 1.3.0 [2017.03.xx] ------------------------------------------------------------------------------ diff --git a/doc/source/connection.rst b/doc/source/connection.rst index 1ae4397..3dc66bb 100644 --- a/doc/source/connection.rst +++ b/doc/source/connection.rst @@ -49,7 +49,7 @@ to numpy `datetime64`/`timedelta64` representation. Conversion options can be also overwritten while executing synchronous/asynchronous queries (:meth:`~qpython.qconnection.QConnection.sync`, -:meth:`~qpython.qconnection.QConnection.async`) or retrieving data from q +:meth:`~qpython.qconnection.QConnection.send`) or retrieving data from q (:meth:`~qpython.qconnection.QConnection.receive`). diff --git a/doc/source/queries.rst b/doc/source/queries.rst index e7b0e93..12d322f 100644 --- a/doc/source/queries.rst +++ b/doc/source/queries.rst @@ -16,7 +16,7 @@ The `qPython` library provides following API methods in the - :func:`~qpython.qconnection.QConnection.sync` - executes a synchronous query against the remote q service, -- :func:`~qpython.qconnection.QConnection.async` - executes an asynchronous +- :func:`~qpython.qconnection.QConnection.send` - executes an asynchronous query against the remote q service, - :func:`~qpython.qconnection.QConnection.query` - executes a query against the remote q service. @@ -65,17 +65,17 @@ Asynchronous queries Calls a anonymous function with a single parameter: - >>> q.async('{til x}', 10) + >>> q.send('{til x}', 10) Executes a q expression: - >>> q.async('til 10') + >>> q.send('til 10') .. note:: The asynchronous query doesn't fetch the result. Query result has to be retrieved explicitly. In order to retrieve query result (for the -:func:`~qpython.qconnection.QConnection.async` or +:func:`~qpython.qconnection.QConnection.send` or :func:`~qpython.qconnection.QConnection.query` methods), one has to call: - :func:`~qpython.qconnection.QConnection.receive` method, which reads next @@ -96,7 +96,7 @@ QMessage: message type: 2, data size: 13, is_compressed: False, data: 10 10 >>> q.sync('asynchMult:{[a;b] res:a*b; (neg .z.w)(res) }') ->>> q.async('asynchMult', 2, 3) +>>> q.send('asynchMult', 2, 3) >>> print(q.receive()) 6 @@ -114,7 +114,7 @@ Type conversions configuration Type conversion options can be overwritten while: - executing synchronous query: :meth:`~qpython.qconnection.QConnection.sync` -- executing asynchronous query: :meth:`~qpython.qconnection.QConnection.async` +- executing asynchronous query: :meth:`~qpython.qconnection.QConnection.send` - retrieving data from q: :meth:`~qpython.qconnection.QConnection.receive` These methods accepts the `options` keywords arguments:: diff --git a/doc/source/usage-examples.rst b/doc/source/usage-examples.rst index 5fdd3c9..a342a4d 100644 --- a/doc/source/usage-examples.rst +++ b/doc/source/usage-examples.rst @@ -124,7 +124,7 @@ Following example presents how to execute simple, asynchronous query against a r a = random.randint(1, 100) b = random.randint(1, 100) print('Asynchronous call with queryid=%s with arguments: %s, %s' % (x, a, b)) - q.async('asynchMult', x, a, b); + q.send('asynchMult', x, a, b); time.sleep(1) finally: diff --git a/qpython/qconnection.py b/qpython/qconnection.py index 6ccefbe..1b0a7bc 100644 --- a/qpython/qconnection.py +++ b/qpython/qconnection.py @@ -309,7 +309,7 @@ def sync(self, query, *parameters, **options): raise QReaderException('Received message of type: %s where response was expected') - def async(self, query, *parameters, **options): + def send(self, query, *parameters, **options): '''Performs an asynchronous query and returns **without** retrieving of the response. @@ -319,11 +319,11 @@ def async(self, query, *parameters, **options): Calls a anonymous function with a single parameter: - >>> q.async('{til x}', 10) + >>> q.send('{til x}', 10) Executes a q expression: - >>> q.async('til 10') + >>> q.send('til 10') :Parameters: - `query` (`string`) - query to be executed diff --git a/samples/async_query.py b/samples/async_query.py index 1f58c8a..f399d96 100644 --- a/samples/async_query.py +++ b/samples/async_query.py @@ -81,7 +81,7 @@ def run(self): a = random.randint(1, 100) b = random.randint(1, 100) print('Asynchronous call with queryid=%s with arguments: %s, %s' % (x, a, b)) - q.async('asynchMult', x, a, b); + q.send('asynchMult', x, a, b); time.sleep(1) finally: From 7257972e1d87aac335b1afd83e4de9a4d3a9933e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B5=D0=BC=D1=91=D0=BD=20=D0=9C=D0=B0=D1=80=D1=8C?= =?UTF-8?q?=D1=8F=D1=81=D0=B8=D0=BD?= Date: Wed, 28 Nov 2018 13:50:18 +0300 Subject: [PATCH 3/3] Fix exception message for wrong response type --- qpython/qconnection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qpython/qconnection.py b/qpython/qconnection.py index 1b0a7bc..317a360 100644 --- a/qpython/qconnection.py +++ b/qpython/qconnection.py @@ -306,7 +306,7 @@ def sync(self, query, *parameters, **options): return response.data else: self._writer.write(QException('nyi: qPython expected response message'), MessageType.ASYNC if response.type == MessageType.ASYNC else MessageType.RESPONSE) - raise QReaderException('Received message of type: %s where response was expected') + raise QReaderException('Received message of type: %s where response was expected' % response.type) def send(self, query, *parameters, **options):