Skip to content

Commit

Permalink
Simplified API for creating QList and QTemporalList instances
Browse files Browse the repository at this point in the history
  • Loading branch information
@maciejlach
maciejlach committed Aug 29, 2014
1 parent d6d8c14 commit 38e60e3
Show file tree
Hide file tree
Showing 7 changed files with 114 additions and 88 deletions.
19 changes: 11 additions & 8 deletions doc/Type-conversion.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,18 +56,21 @@ def qlist(array, adjust_dtype = True, **meta)

which simplifies creation of `QList` objects. This method:
- creates a `numpy` view `QList`
- converts the input `numpy` array if the original `dtype` doesn’t match requested conversion, e.g.: `numpy.int16` array requested to be represented as a `QLONG_LIST`
- if required, creates `numpy` array from the input `array`
- converts the input `numpy` array if the original `dtype` doesn't match requested conversion, e.g.: `numpy.int16` array requested to be represented as a `QLONG_LIST`
- optional meta information, e.g.: q type indicator: `qtype=QLONG_LIST`

For example:
```python
# 1 2 3
qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST)
qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST)
qlist([1, 2, 3], qtype = QLONG_LIST)

# (0x01;0x02;0xff)
qlist(numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), qtype = QBYTE_LIST)
qlist(numpy.array([0x01, 0x02, 0xff]), qtype = QBYTE_LIST)
qlist([0x01, 0x02, 0xff], qtype = QBYTE_LIST)
```

### String and symbols
Expand Down Expand Up @@ -101,20 +104,20 @@ Atom q temporal types are represented as instances of `QTemporal` class, which i

Lists of temporal values are represented as instances of `QTemporalList` class. This class wraps the raw q representation of temporal data (i.e. `long`s for `timestamp`s, `int`s for `month`s etc.) and provides accessors which allow to convert raw data to `QTemporal` instances in a lazy manner.

In addition, the `qtemporal` module provides an utility method, which converts a `numpy.array` to q temporal list and enriches object instance with provided meta data:
The `qcollection` utility method: `qlist` converts an input `array` to q temporal list (view on `numpy.array`) and enriches object instance with provided meta data:

```python
def qtemporallist(array, **meta)
def qlist(array, **meta)
```

For example:
```python
# 2001.01.01 2000.05.01 0Nd
qtemporallist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), to_raw_qtemporal(numpy.datetime64('2000-05-01', 'D'), qtype=QDATE), qnull(QDATE)]), qtype=QDATE_LIST)
qtemporallist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)
qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), to_raw_qtemporal(numpy.datetime64('2000-05-01', 'D'), qtype=QDATE), qnull(QDATE)]), qtype=QDATE_LIST)
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)

# 2000.01.04D05:36:57.600 0Np
qtemporallist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST)
qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST)
```


Expand Down Expand Up @@ -161,7 +164,7 @@ qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST),
# ([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))
qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
qtemporallist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)])
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)])
```

The keyed tables are represented via `QKeyedTable` instances, in which both keys and values are stored as a separate `QTable` instances.
Expand All @@ -173,7 +176,7 @@ QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]),
qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
qtemporallist(numpy.array([366, 121, qnull(QDATE)]), qtype = QDATE_LIST)]))
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype = QDATE_LIST)]))
```

### Lambdas
Expand Down
29 changes: 26 additions & 3 deletions qpython/qcollection.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
import numpy

from qpython import MetaData
from qpython.qtype import QTABLE, FROM_Q
from qpython.qtype import QTABLE, FROM_Q, QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN
from qpython.qtemporal import from_raw_qtemporal, to_raw_qtemporal


class QList(numpy.ndarray):
Expand All @@ -36,15 +37,37 @@ def __hash__(self):



class QTemporalList(QList):
'''Represents a q list of datetime objects.'''
def meta_init(self, **meta):
self.meta = MetaData(**meta)

def __getitem__(self, idx):
return from_raw_qtemporal(numpy.ndarray.__getitem__(self, idx), -self.meta.qtype)

def __setitem__(self, idx, value):
numpy.ndarray.__setitem__(self, idx, to_raw_qtemporal(value, -self.meta.qtype))

def raw(self, idx):
return numpy.ndarray.__getitem__(self, idx)



def qlist(array, adjust_dtype = True, **meta):
'''Converts a numpy.array to q vector and enriches object instance with given meta data.'''
'''Converts an input array to q vector and enriches object instance with given meta data. If necessary input array is converted from tuple or list to numpy.array.'''
if type(array) in (list, tuple):
array = numpy.array(array)

if type(array) != numpy.ndarray:
raise ValueError('array parameter is expected to be of type: numpy.ndarray, list or tuple')

if meta and 'qtype' in meta:
qtype = -abs(meta['qtype'])
dtype = FROM_Q[qtype]
if dtype != array.dtype:
array = array.astype(dtype = dtype)

vector = array.view(QList)
vector = array.view(QList) if not qtype in [QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN] else array.view(QTemporalList)
vector.meta_init(**meta)
return vector

Expand Down
4 changes: 2 additions & 2 deletions qpython/qreader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import qlist, QDictionary, qtable, QTable, QKeyedTable
from qpython.qtemporal import qtemporallist, from_raw_qtemporal
from qpython.qtemporal import from_raw_qtemporal

try:
from qpython.fastutils import uncompress
Expand Down Expand Up @@ -232,7 +232,7 @@ def _read_list(self, qtype):
data = numpy.fromstring(raw, dtype = conversion)
if not self._is_native:
data.byteswap(True)
return qtemporallist(data, qtype = qtype, adjust_dtype = False)
return qlist(data, qtype = qtype, adjust_dtype = False)
elif qtype == QGUID_LIST:
data = numpy.array([self._read_guid() for x in xrange(length)])
return qlist(data, qtype = qtype, adjust_dtype = False)
Expand Down
31 changes: 0 additions & 31 deletions qpython/qtemporal.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
#

from qpython import MetaData
from qcollection import QList
from qtype import * # @UnusedWildImport


Expand Down Expand Up @@ -63,36 +62,6 @@ def __ne__(self, other):



class QTemporalList(QList):
'''Represents a q list of datetime objects.'''
def meta_init(self, **meta):
self.meta = MetaData(**meta)

def __getitem__(self, idx):
return from_raw_qtemporal(numpy.ndarray.__getitem__(self, idx), -self.meta.qtype)

def __setitem__(self, idx, value):
numpy.ndarray.__setitem__(self, idx, to_raw_qtemporal(value, -self.meta.qtype))

def raw(self, idx):
return numpy.ndarray.__getitem__(self, idx)



def qtemporallist(array, **meta):
'''Converts a numpy.array to q temporal list and enriches object instance with given meta data.'''
if meta and 'qtype' in meta:
qtype = -abs(meta['qtype'])
dtype = FROM_Q[qtype]
if dtype != array.dtype:
array = array.astype(dtype = dtype)

result = array.view(QTemporalList)
result.meta_init(**meta)
return result



def qtemporal(dt, **meta):
'''Converts a numpy.datetime64 to q temporal object and enriches object instance with given meta data.'''
result = QTemporal(dt)
Expand Down
5 changes: 2 additions & 3 deletions qpython/qwriter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,8 @@
import sys

from qtype import * # @UnusedWildImport
from qcollection import QList, QDictionary, QTable
from qpython.qtemporal import QTemporalList, QTemporal, to_raw_qtemporal
from qpython.qcollection import qlist, QKeyedTable
from qcollection import qlist, QList, QTemporalList, QDictionary, QTable, QKeyedTable
from qpython.qtemporal import QTemporal, to_raw_qtemporal


class QWriterException(Exception):
Expand Down
24 changes: 12 additions & 12 deletions tests/qreader_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
from collections import OrderedDict
from qpython import qreader
from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import qlist, QList, QDictionary, qtable, QKeyedTable
from qpython.qtemporal import qtemporal, QTemporal, qtemporallist, QTemporalList
from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, qtable, QKeyedTable
from qpython.qtemporal import qtemporal, QTemporal



Expand All @@ -32,28 +32,28 @@
qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST)),
('"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')),
('"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')),
('(2001.01m; 0Nm)', qtemporallist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST)),
('(2001.01m; 0Nm)', qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST)),
('2001.01m', qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH)),
('0Nm', qtemporal(qnull(QMONTH), qtype=QMONTH)),
('2001.01.01 2000.05.01 0Nd', qtemporallist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)),
('2001.01.01 2000.05.01 0Nd', qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)),
('2001.01.01', qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE)),
('0Nd', qtemporal(qnull(QDATE), qtype=QDATE)),
('2000.01.04T05:36:57.600 0Nz', qtemporallist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST)),
('2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST)),
('2000.01.04T05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), qtype=QDATETIME)),
('0Nz', qtemporal(qnull(QDATETIME), qtype=QDATETIME)),
('12:01 0Nu', qtemporallist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST)),
('12:01 0Nu', qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST)),
('12:01', qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE)),
('0Nu', qtemporal(qnull(QMINUTE), qtype=QMINUTE)),
('12:05:00 0Nv', qtemporallist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST)),
('12:05:00 0Nv', qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST)),
('12:05:00', qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND)),
('0Nv', qtemporal(qnull(QSECOND), qtype=QSECOND)),
('12:04:59.123 0Nt', qtemporallist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST)),
('12:04:59.123 0Nt', qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST)),
('12:04:59.123', qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME)),
('0Nt', qtemporal(qnull(QTIME), qtype=QTIME)),
('2000.01.04D05:36:57.600 0Np', qtemporallist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST)),
('2000.01.04D05:36:57.600 0Np', qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST)),
('2000.01.04D05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), qtype=QTIMESTAMP)),
('0Np', qtemporal(qnull(QTIMESTAMP), qtype=QTIMESTAMP)),
('0D05:36:57.600 0Nn', qtemporallist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST)),
('0D05:36:57.600 0Nn', qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST)),
('0D05:36:57.600', qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN)),
('0Nn', qtemporal(qnull(QTIMESPAN), qtype=QTIMESPAN)),

Expand Down Expand Up @@ -143,13 +143,13 @@
('([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
qtemporallist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)])),
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)])),
('([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]),
qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
qtemporallist(numpy.array([366, 121, qnull(QDATE)]), qtype = QDATE_LIST)]))
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype = QDATE_LIST)]))
),

))
Expand Down
Loading

0 comments on commit 38e60e3

Please sign in to comment.