From 27e6bc39b9e246c4c8296244d89b958818921080 Mon Sep 17 00:00:00 2001 From: maciejlach Date: Wed, 24 Sep 2014 10:23:01 +0200 Subject: [PATCH] Redesign lambdas and projections handling Fixes #7: validate lambda expression upon construction --- CHANGELOG.txt | 6 +- doc/source/type-conversion.rst | 26 +++-- qpython/qreader.py | 43 ++++++-- qpython/qtype.py | 193 ++++++++++++++++++++++----------- qpython/qwriter.py | 41 ++++--- tests/QExpressions3.out | 20 ++++ tests/qreader_test.py | 16 ++- tests/qwriter_test.py | 2 +- 8 files changed, 241 insertions(+), 106 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 4e1a7f0..a83a15d 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,7 +1,8 @@ ------------------------------------------------------------------------------ - qPython 1.0 Beta 5 [TBA] + qPython 1.0 Beta 5 [2014.10.01] ------------------------------------------------------------------------------ + - Redesign lambdas and projections handling - Rework collections API - Extend QConnection class with context manager API - Fix: reading of generic null (::) embedded in lists @@ -11,7 +12,8 @@ qPython 1.0 Beta 4 [2014.07.04] ------------------------------------------------------------------------------ - - qtemporallist: force numpy.array type conversion in case of mismatch between meta.qtype and dtype of raw list + - qtemporallist: force numpy.array type conversion in case of mismatch between + meta.qtype and dtype of raw list - Enable Travis CI integration - Update project meta-information diff --git a/doc/source/type-conversion.rst b/doc/source/type-conversion.rst index 040d0f9..20b17b8 100644 --- a/doc/source/type-conversion.rst +++ b/doc/source/type-conversion.rst @@ -261,16 +261,28 @@ For example:: qlist(numpy.array([366, 121, qnull(QDATE)]), qtype = QDATE_LIST)])) -Lambdas -******* +Functions, lambdas and projections +********************************** -The q lambda is mapped to custom Python class :class:`.qtype.QLambda`:: +IPC protocol type codes 100+ are used to represent functions, lambdas and +projections. These types are represented as instances of base class +:class:`.qtype.QFunction` or descendent classes: - # {x+y} - QLambda('{x+y}') +- :class:`.qtype.QLambda` - represents q lambda expression, note the expression + is required to be either: - # {x+y} [3] - QLambda('{x+y}', numpy.int64(3)) + - q expression enclosed in {}, e.g.: ``{x + y}`` + - k expression, e.g.: ``k){x + y}`` + +- :class:`.qtype.QProjection` - represents function projection with parameters:: + + # { x + y}[3] + QProjection([QLambda('{x+y}'), numpy.int64(3)]) + + +.. note:: Only :class:`.qtype.QLambda` and :class:`.qtype.QProjection` are + serializable. qPython doesn't provide means to serialize other + function types. Errors diff --git a/qpython/qreader.py b/qpython/qreader.py index 49c6670..b3b9f58 100644 --- a/qpython/qreader.py +++ b/qpython/qreader.py @@ -205,12 +205,6 @@ def _read_object(self): raise QReaderException('Unable to deserialize q type: %s' % hex(qtype)) - @parse(QNULL) - def _read_null(self, qtype = QNULL): - self._buffer.get_byte() # ignore - return None - - @parse(QERROR) def _read_error(self, qtype = QERROR): raise QException(self._read_symbol()) @@ -315,6 +309,15 @@ def _read_general_list(self, qtype = QGENERAL_LIST): return [self._read_object() for x in xrange(length)] + @parse(QNULL) + @parse(QUNARY_FUNC) + @parse(QBINARY_FUNC) + @parse(QTERNARY_FUNC) + def _read_function(self, qtype = QNULL): + code = self._buffer.get_byte() + return None if qtype == QNULL and code == 0 else QFunction(qtype) + + @parse(QLAMBDA) def _read_lambda(self, qtype = QLAMBDA): self._buffer.get_symbol() # skip @@ -322,12 +325,28 @@ def _read_lambda(self, qtype = QLAMBDA): return QLambda(expression) - @parse(QLAMBDA_PART) - def _read_lambda_part(self, qtype = QLAMBDA): - length = self._buffer.get_int() - 1 - qlambda = self._read_lambda(qtype) - qlambda.parameters = [ self._read_object() for x in range(length) ] - return qlambda + @parse(QCOMPOSITION_FUNC) + def _read_function_composition(self, qtype = QCOMPOSITION_FUNC): + self._read_projection(qtype) # skip + return QFunction(qtype) + + + @parse(QADVERB_FUNC_106) + @parse(QADVERB_FUNC_107) + @parse(QADVERB_FUNC_108) + @parse(QADVERB_FUNC_109) + @parse(QADVERB_FUNC_110) + @parse(QADVERB_FUNC_111) + def _read_adverb_function(self, qtype = QADVERB_FUNC_106): + self._read_object() # skip + return QFunction(qtype) + + + @parse(QPROJECTION) + def _read_projection(self, qtype = QPROJECTION): + length = self._buffer.get_int() + parameters = [ self._read_object() for x in range(length) ] + return QProjection(parameters) def _read_bytes(self, length): diff --git a/qpython/qtype.py b/qpython/qtype.py index bdd51e7..945f152 100644 --- a/qpython/qtype.py +++ b/qpython/qtype.py @@ -70,63 +70,85 @@ QKEYED_TABLE 0x63 QTABLE 0x62 QLAMBDA 0x64 -QLAMBDA_PART 0x68 +QUNARY_FUNC 0x65 +QBINARY_FUNC 0x66 +QTERNARY_FUNC 0x67 +QCOMPOSITION_FUNC 0x69 +QADVERB_FUNC_106 0x6a +QADVERB_FUNC_107 0x6b +QADVERB_FUNC_108 0x6c +QADVERB_FUNC_109 0x6d +QADVERB_FUNC_110 0x6e +QADVERB_FUNC_111 0x6f +QPROJECTION 0x68 QERROR -0x80 ================== ============= ''' import numpy +import re import uuid # qtype constants: -QNULL = 0x65 -QGENERAL_LIST = 0x00 -QBOOL = -0x01 -QBOOL_LIST = 0x01 -QGUID = -0x02 -QGUID_LIST = 0x02 -QBYTE = -0x04 -QBYTE_LIST = 0x04 -QSHORT = -0x05 -QSHORT_LIST = 0x05 -QINT = -0x06 -QINT_LIST = 0x06 -QLONG = -0x07 -QLONG_LIST = 0x07 -QFLOAT = -0x08 -QFLOAT_LIST = 0x08 -QDOUBLE = -0x09 -QDOUBLE_LIST = 0x09 -QCHAR = -0x0a -QSTRING = 0x0a -QSTRING_LIST = 0x00 -QSYMBOL = -0x0b -QSYMBOL_LIST = 0x0b - -QTIMESTAMP = -0x0c -QTIMESTAMP_LIST = 0x0c -QMONTH = -0x0d -QMONTH_LIST = 0x0d -QDATE = -0x0e -QDATE_LIST = 0x0e -QDATETIME = -0x0f -QDATETIME_LIST = 0x0f -QTIMESPAN = -0x10 -QTIMESPAN_LIST = 0x10 -QMINUTE = -0x11 -QMINUTE_LIST = 0x11 -QSECOND = -0x12 -QSECOND_LIST = 0x12 -QTIME = -0x13 -QTIME_LIST = 0x13 - -QDICTIONARY = 0x63 -QKEYED_TABLE = 0x63 -QTABLE = 0x62 -QLAMBDA = 0x64 -QLAMBDA_PART = 0x68 -QERROR = -0x80 +QNULL = 0x65 +QGENERAL_LIST = 0x00 +QBOOL = -0x01 +QBOOL_LIST = 0x01 +QGUID = -0x02 +QGUID_LIST = 0x02 +QBYTE = -0x04 +QBYTE_LIST = 0x04 +QSHORT = -0x05 +QSHORT_LIST = 0x05 +QINT = -0x06 +QINT_LIST = 0x06 +QLONG = -0x07 +QLONG_LIST = 0x07 +QFLOAT = -0x08 +QFLOAT_LIST = 0x08 +QDOUBLE = -0x09 +QDOUBLE_LIST = 0x09 +QCHAR = -0x0a +QSTRING = 0x0a +QSTRING_LIST = 0x00 +QSYMBOL = -0x0b +QSYMBOL_LIST = 0x0b + +QTIMESTAMP = -0x0c +QTIMESTAMP_LIST = 0x0c +QMONTH = -0x0d +QMONTH_LIST = 0x0d +QDATE = -0x0e +QDATE_LIST = 0x0e +QDATETIME = -0x0f +QDATETIME_LIST = 0x0f +QTIMESPAN = -0x10 +QTIMESPAN_LIST = 0x10 +QMINUTE = -0x11 +QMINUTE_LIST = 0x11 +QSECOND = -0x12 +QSECOND_LIST = 0x12 +QTIME = -0x13 +QTIME_LIST = 0x13 + +QDICTIONARY = 0x63 +QKEYED_TABLE = 0x63 +QTABLE = 0x62 +QLAMBDA = 0x64 +QUNARY_FUNC = 0x65 +QBINARY_FUNC = 0x66 +QTERNARY_FUNC = 0x67 +QCOMPOSITION_FUNC = 0x69 +QADVERB_FUNC_106 = 0x6a +QADVERB_FUNC_107 = 0x6b +QADVERB_FUNC_108 = 0x6c +QADVERB_FUNC_109 = 0x6d +QADVERB_FUNC_110 = 0x6e +QADVERB_FUNC_111 = 0x6f +QPROJECTION = 0x68 + +QERROR = -0x80 @@ -269,34 +291,82 @@ class QException(Exception): -class QLambda(object): +class QFunction(object): + '''Represents a q function.''' + + def __init__(self, qtype): + self.qtype = qtype + + + def __str__(self): + return '%s#%s' % (self.__class__.__name__, self.qtype) + + + +class QLambda(QFunction): + '''Represents a q lambda expression. + .. note:: `expression` is trimmed and required to be valid q function + (``{..}``) or k function (``k){..}``). + :Parameters: - `expression` (`string`) - lambda expression - - `parameters` (`list`) - list of parameters for lambda expression + + :raises: `ValueError` ''' - def __init__(self, expression, *parameters): + def __init__(self, expression): + QFunction.__init__(self, QLAMBDA) + + if not expression: + raise ValueError('Lambda expression cannot be None or empty') + + expression = expression.strip() + + if not QLambda._EXPRESSION_REGEX.match(expression): + raise ValueError('Invalid lambda expression: %s' % expression) + self.expression = expression + + + _EXPRESSION_REGEX = re.compile(r'\s*(k\))?\s*\{.*\}') + + + def __str__(self): + return '%s(\'%s\')' % (self.__class__.__name__, self.expression) + + + def __eq__(self, other): + return (isinstance(other, self.__class__) and self.expression == other.expression) + + + +class QProjection(QFunction): + + '''Represents a q projection. + + :Parameters: + - `parameters` (`list`) - list of parameters for lambda expression + ''' + def __init__(self, parameters): + QFunction.__init__(self, QPROJECTION) + self.parameters = parameters + def __str__(self): - params = [] + parameters_str = [] for arg in self.parameters: - params.append('%s' % arg) - - return '%s(\'%s\'%s)' % (self.__class__.__name__, self.expression, ', '.join(params) if params else '') + parameters_str.append('%s' % arg) + + return '%s(%s)' % (self.__class__.__name__, ', '.join(parameters_str)) + def __eq__(self, other): - result = (isinstance(other, self.__class__) and self.expression == other.expression) - - if not result: - return False - return (not self.parameters and not other.parameters) or \ reduce(lambda v1,v2: v1 or v2, map(lambda v: v in self.parameters, other.parameters)) - + def __ne__(self, other): return not self.__eq__(other) @@ -311,6 +381,7 @@ class Mapper(object): def __init__(self, call_map): self.call_map = call_map + def __call__(self, *args): def wrap(func): for arg in args: diff --git a/qpython/qwriter.py b/qpython/qwriter.py index 8b5689f..f680f32 100644 --- a/qpython/qwriter.py +++ b/qpython/qwriter.py @@ -1,18 +1,18 @@ -# +# # Copyright (c) 2011-2014 Exxeleron GmbH -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# +# import cStringIO import struct @@ -154,7 +154,7 @@ def _write_symbol(self, data): self._buffer.write(data) self._buffer.write('\0') - + @serialize(uuid.UUID) def _write_guid(self, data): self._buffer.write(struct.pack('=b', QGUID)) @@ -166,7 +166,7 @@ def _write_temporal(self, data): try: if self.protocol_version < 1 and (data.meta.qtype == QTIMESPAN or data.meta.qtype == QTIMESTAMP): raise QWriterException('kdb+ protocol version violation: data type %s not supported pre kdb+ v2.6' % hex(data.meta.qtype)) - + self._buffer.write(struct.pack('=b', data.meta.qtype)) fmt = STRUCT_MAP[data.meta.qtype] self._buffer.write(struct.pack(fmt, to_raw_qtemporal(data.raw, data.meta.qtype))) @@ -176,17 +176,16 @@ def _write_temporal(self, data): @serialize(QLambda) def _write_lambda(self, data): - if not data.parameters: - self._buffer.write(struct.pack('=b', QLAMBDA)) - self._buffer.write('\0') - self._write_string(data.expression) - else: - self._buffer.write(struct.pack('=bi', QLAMBDA_PART, len(data.parameters) + 1)) - self._buffer.write(struct.pack('=b', QLAMBDA)) - self._buffer.write('\0') - self._write_string(data.expression) - for parameter in data.parameters: - self._write(parameter) + self._buffer.write(struct.pack('=b', QLAMBDA)) + self._buffer.write('\0') + self._write_string(data.expression) + + + @serialize(QProjection) + def _write_projection(self, data): + self._buffer.write(struct.pack('=bi', QPROJECTION, len(data.parameters))) + for parameter in data.parameters: + self._write(parameter) @serialize(QDictionary, QKeyedTable) @@ -209,10 +208,10 @@ def _write_table(self, data): def _write_list(self, data, qtype = None): if qtype is not None: qtype = -abs(qtype) - + if qtype is None: qtype = get_list_qtype(data) - + if self.protocol_version < 1 and (data.meta.qtype == QTIMESPAN_LIST or data.meta.qtype == QTIMESTAMP_LIST): raise QWriterException('kdb+ protocol version violation: data type %s not supported pre kdb+ v2.6' % hex(data.meta.qtype)) @@ -228,7 +227,7 @@ def _write_list(self, data, qtype = None): elif qtype == QGUID: if self.protocol_version < 3: raise QWriterException('kdb+ protocol version violation: Guid not supported pre kdb+ v3.0') - + for guid in data: self._buffer.write(guid.bytes) else: diff --git a/tests/QExpressions3.out b/tests/QExpressions3.out index 49939bc..11c0df5 100644 --- a/tests/QExpressions3.out +++ b/tests/QExpressions3.out @@ -142,6 +142,26 @@ ED00000080 64000A00050000007B782B797D {x+y}[3] 680200000064000A00050000007B782B797DF90300000000000000 +insert [1] +6802000000661cf90100000000000000 +xbar +6471000a00240000006b297b782a792064697620783a245b3136683d6162735b40785d3b226a2224783b785d7d +not +650f +and +6605 +md5 +68020000006610f9f1ffffffffffffff +any +6902000000651c6802000000660bf662 +save +6a6471000a003c0000006b297b245b313d23703a605c3a2a7c605c3a783a2d3121783b7365745b783b2e202a705d3b2020207820303a2e682e74785b7020315d402e2a705d7d +raze +6b660c +sums +6c6601 +prev +6d6600 (enlist `a)!(enlist 1) 630B000100000061000700010000000100000000000000 1 2!`abc`cdefgh diff --git a/tests/qreader_test.py b/tests/qreader_test.py index c69d377..9a44126 100644 --- a/tests/qreader_test.py +++ b/tests/qreader_test.py @@ -99,8 +99,18 @@ ('("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")', ['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog']), ('{x+y}', QLambda('{x+y}')), - ('{x+y}[3]', QLambda('{x+y}', numpy.int64(3))), - + ('{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)])), + ('insert [1]', QProjection([QFunction(0), numpy.int64(1)])), + ('xbar', QLambda('k){x*y div x:$[16h=abs[@x];"j"$x;x]}')), + ('not', QFunction(0)), + ('and', QFunction(0)), + ('md5', QProjection([QFunction(0), numpy.int64(-15)])), + ('any', QFunction(0)), + ('save', QFunction(0)), + ('raze', QFunction(0)), + ('sums', QFunction(0)), + ('prev', QFunction(0)), + ('(enlist `a)!(enlist 1)', QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST), qlist(numpy.array([1], dtype=numpy.int64), qtype=QLONG_LIST))), ('1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST), @@ -203,6 +213,8 @@ def compare(left, right): return numpy.isnan(right.raw) elif type(left) in [list, tuple, numpy.ndarray, QList, QTemporalList]: return arrays_equal(left, right) + elif type(left) == QFunction: + return type(right) == QFunction else: return left == right diff --git a/tests/qwriter_test.py b/tests/qwriter_test.py index 21562b8..1086f92 100644 --- a/tests/qwriter_test.py +++ b/tests/qwriter_test.py @@ -146,7 +146,7 @@ qlist(numpy.array(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog']), qtype = QSTRING_LIST), qlist(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog'], qtype = QSTRING_LIST))), ('{x+y}', QLambda('{x+y}')), - ('{x+y}[3]', QLambda('{x+y}', numpy.int64(3))), + ('{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)])), ('(enlist `a)!(enlist 1)', (QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST), qlist(numpy.array([1], dtype=numpy.int64), qtype=QLONG_LIST)),