From 980089ab7a171ece1477adeb0a14403f8d12e75d Mon Sep 17 00:00:00 2001 From: Maciej Lach Date: Tue, 12 Jan 2016 19:35:35 +0100 Subject: [PATCH] API redesign: enable custom serializers & deserializers (relates to #35) --- CHANGELOG.txt | 6 ++++++ doc/source/conf.py | 2 +- doc/source/connection.rst | 44 +++++++++++++++++++++++++++++++++++++-- qpython/__init__.py | 2 +- qpython/qconnection.py | 24 ++++++++++++++++++--- qpython/qreader.py | 15 ++----------- qpython/qwriter.py | 12 ----------- tests/pandas_test.py | 7 ++++--- 8 files changed, 77 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 9dc7877..2713bac 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,9 @@ +------------------------------------------------------------------------------ + qPython 1.2.0 [2016.01.*] +------------------------------------------------------------------------------ + + - API redesign: enable custom serializers & deserializers + ------------------------------------------------------------------------------ qPython 1.1.0 [2015.11.25] ------------------------------------------------------------------------------ diff --git a/doc/source/conf.py b/doc/source/conf.py index f3600b3..bb190ec 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -64,7 +64,7 @@ def __getattr__(cls, name): # General information about the project. project = u'qPython' -copyright = u'2014-2015, DEVnet' +copyright = u'2014-2016, DEVnet' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/doc/source/connection.rst b/doc/source/connection.rst index 0eaf908..aa7817e 100644 --- a/doc/source/connection.rst +++ b/doc/source/connection.rst @@ -13,7 +13,7 @@ qPython wraps connection to a q process in instances of the q.close() .. note:: the connection is not established when the connector instance is - created. The connection is initialized explicitly by calling the + created. The connection is initialised explicitly by calling the :meth:`~qpython.qconnection.QConnection.open` method. @@ -48,4 +48,44 @@ to numpy `datetime64`/`timedelta64` representation. Conversion options can be also overwritten while executing synchronous/asynchronous queries (:meth:`~qpython.qconnection.QConnection.sync`, :meth:`~qpython.qconnection.QConnection.async`) or retrieving data from q -(:meth:`~qpython.qconnection.QConnection.receive`). \ No newline at end of file +(:meth:`~qpython.qconnection.QConnection.receive`). + + +Custom IPC protocol serializers/deserializers +********************************************* + +Default IPC serializers (`.QWriter` and `.PandasQWriter`) and deserializers +(`.QReader` and `.PandasQReader`) can be replaced with custom implementations. +This allow users to override the default mapping between the q types and Python +representation. +:: + + q = qconnection.QConnection(host = 'localhost', port = 5000, writer_class = MyQWriter, reader_class = MyQReader) + + +:: + + class MyQReader(QReader): + # QReader and QWriter use decorators to map data types and corresponding function handlers + parse = Mapper(QReader._reader_map) + + def _read_list(self, qtype): + if qtype == QSYMBOL_LIST: + self._buffer.skip() + length = self._buffer.get_int() + symbols = self._buffer.get_symbols(length) + return [s.decode(self._encoding) for s in symbols] + else: + return QReader._read_list(self, qtype = qtype) + + @parse(QSYMBOL) + def _read_symbol(self, qtype = QSYMBOL): + return numpy.string_(self._buffer.get_symbol()).decode(self._encoding) + + + with qconnection.QConnection(host='localhost', port=5000, reader_class = MyQReader) as q: + symbols = q.sync('`foo`bar') + print(symbols, type(symbols), type(symbols[0])) + + symbol = q.sync('`foo') + print(symbol, type(symbol)) diff --git a/qpython/__init__.py b/qpython/__init__.py index 3abaebc..ba4f67f 100644 --- a/qpython/__init__.py +++ b/qpython/__init__.py @@ -17,7 +17,7 @@ __all__ = ['qconnection', 'qtype', 'qtemporal', 'qcollection'] -__version__ = '1.1.0' +__version__ = '1.2.0b1' diff --git a/qpython/qconnection.py b/qpython/qconnection.py index 162b357..4fd0399 100644 --- a/qpython/qconnection.py +++ b/qpython/qconnection.py @@ -62,6 +62,9 @@ class QConnection(object): - `username` (`string` or `None`) - username for q authentication/authorization - `password` (`string` or `None`) - password for q authentication/authorization - `timeout` (`nonnegative float` or `None`) - set a timeout on blocking socket operations + - `encoding` (`string`) - string encoding for data deserialization + - `reader_class` (subclass of `QReader`) - data deserializer + - `writer_class` (subclass of `QWriter`) - data serializer :Options: - `raw` (`boolean`) - if ``True`` returns raw data chunk instead of parsed data, **Default**: ``False`` @@ -74,7 +77,8 @@ class QConnection(object): strings are encoded as q strings instead of chars, **Default**: ``False`` ''' - def __init__(self, host, port, username = None, password = None, timeout = None, encoding = 'latin-1', **options): + + def __init__(self, host, port, username = None, password = None, timeout = None, encoding = 'latin-1', reader_class = None, writer_class = None, **options): self.host = host self.port = port self.username = username @@ -89,6 +93,20 @@ def __init__(self, host, port, username = None, password = None, timeout = None, self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) + try: + from qpython._pandas import PandasQReader, PandasQWriter + self._reader_class = PandasQReader + self._writer_class = PandasQWriter + except ImportError: + self._reader_class = QReader + self._writer_class = QWriter + + if reader_class: + self._reader_class = reader_class + + if writer_class: + self._writer_class = writer_class + def __enter__(self): self.open() @@ -124,8 +142,8 @@ def open(self): self._init_socket() self._initialize() - self._writer = QWriter(self._connection, protocol_version = self._protocol_version) - self._reader = QReader(self._connection.makefile('b')) + self._writer = self._writer_class(self._connection, protocol_version = self._protocol_version) + self._reader = self._reader_class(self._connection.makefile('b')) def _init_socket(self): diff --git a/qpython/qreader.py b/qpython/qreader.py index bf6b13b..3df3f00 100644 --- a/qpython/qreader.py +++ b/qpython/qreader.py @@ -99,24 +99,13 @@ class QReader(object): :Parameters: - `stream` (`file object` or `None`) - data input stream + - `encoding` (`string`) - encoding for characters parsing ''' _reader_map = {} parse = Mapper(_reader_map) - def __new__(cls, *args, **kwargs): - if cls is QReader: - # try to load optional pandas binding - try: - from qpython._pandas import PandasQReader - return super(QReader, cls).__new__(PandasQReader) - except ImportError: - return super(QReader, cls).__new__(QReader) - else: - return super(QReader, cls).__new__(cls) - - def __init__(self, stream, encoding = 'latin-1'): self._stream = stream self._buffer = QReader.BytesBuffer() @@ -257,7 +246,7 @@ def _read_symbol(self, qtype = QSYMBOL): @parse(QCHAR) def _read_char(self, qtype = QCHAR): - return chr(self._read_atom(QCHAR)).encode(self._encoding) + return chr(self._read_atom(QCHAR)).encode(self._encoding) @parse(QGUID) diff --git a/qpython/qwriter.py b/qpython/qwriter.py index e4a2d0a..58e6fb4 100644 --- a/qpython/qwriter.py +++ b/qpython/qwriter.py @@ -50,18 +50,6 @@ class QWriter(object): serialize = Mapper(_writer_map) - def __new__(cls, *args, **kwargs): - if cls is QWriter: - # try to load optional pandas binding - try: - from qpython._pandas import PandasQWriter - return super(QWriter, cls).__new__(PandasQWriter) - except ImportError: - return super(QWriter, cls).__new__(QWriter) - else: - return super(QWriter, cls).__new__(cls) - - def __init__(self, stream, protocol_version): self._stream = stream self._protocol_version = protocol_version diff --git a/tests/pandas_test.py b/tests/pandas_test.py index 12b729d..ded2bf9 100644 --- a/tests/pandas_test.py +++ b/tests/pandas_test.py @@ -23,7 +23,8 @@ from io import BytesIO from collections import OrderedDict -from qpython import qreader, MetaData, qwriter +from qpython import MetaData +from qpython._pandas import PandasQReader, PandasQWriter from qpython.qtype import * # @UnusedWildImport from qpython.qcollection import qlist, QList, QTemporalList, QDictionary from qpython.qtemporal import QTemporal @@ -250,7 +251,7 @@ def test_reading_pandas(): sys.stdout.write(' %-75s' % query) try: buffer_.seek(0) - stream_reader = qreader.QReader(buffer_) + stream_reader = PandasQReader(buffer_) result = stream_reader.read(pandas = True).data if isinstance(value, dict): if 'index' in value: @@ -271,7 +272,7 @@ def test_reading_pandas(): def test_writing_pandas(): - w = qwriter.QWriter(None, 3) + w = PandasQWriter(None, 3) for query, value in iter(PANDAS_EXPRESSIONS.items()): sys.stdout.write( '%-75s' % query )