Skip to content

Commit

Permalink
Switch to raw millis/nano represenation of epochs
Browse files Browse the repository at this point in the history
  • Loading branch information
@maciejlach
maciejlach committed Nov 5, 2014
1 parent 483835c commit cce25b6
Show file tree
Hide file tree
Showing 5 changed files with 38 additions and 41 deletions.
27 changes: 12 additions & 15 deletions qpython/qtemporal.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,17 @@
from qpython import MetaData
from qtype import * # @UnusedWildImport

import pandas as pd

_MILLIS_PER_DAY = 24 * 60 * 60 * 1000
_MILLIS_PER_DAY_FLOAT = float(_MILLIS_PER_DAY)
_QEPOCH_MS = long(10957 * _MILLIS_PER_DAY)
_EPOCH_QTIMESTAMP_NS = _QEPOCH_MS * 1000000

_MILIS_PER_DAY = 24 * 60 * 60 * 1000

_EPOCH_QMONTH = numpy.datetime64('2000-01', 'M')
_EPOCH_QDATE = numpy.datetime64('2000-01-01', 'D')
_EPOCH_QDATETIME = pd.Timestamp('2000-01-01 00:00:00.000')
_EPOCH_TIMESTAMP = pd.Timestamp('2000-01-01 00:00:00')
_EPOCH_QDATETIME = numpy.datetime64(_QEPOCH_MS, 'ms')
_EPOCH_TIMESTAMP = numpy.datetime64(_EPOCH_QTIMESTAMP_NS, 'ns')


_QMONTH_NULL = qnull(QMONTH)
Expand Down Expand Up @@ -258,7 +260,7 @@ def _from_qdatetime(raw):
if numpy.isnan(raw) or raw == _QDATETIME_NULL:
return _NUMPY_NULL[QDATETIME]
else:
return _EPOCH_QDATETIME + numpy.timedelta64(long(_MILIS_PER_DAY * raw), 'ms')
return _EPOCH_QDATETIME + numpy.timedelta64(long(_MILLIS_PER_DAY * raw), 'ms')



Expand All @@ -267,7 +269,7 @@ def _to_qdatetime(dt):
if t_dt == numpy.float64:
return dt
elif t_dt == numpy.datetime64:
return (dt - _EPOCH_QDATETIME).astype(float) / _MILIS_PER_DAY if not dt == _NUMPY_NULL[QDATETIME] else _QDATETIME_NULL
return (dt - _EPOCH_QDATETIME).astype(float) / _MILLIS_PER_DAY if not dt == _NUMPY_NULL[QDATETIME] else _QDATETIME_NULL
else:
raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

Expand Down Expand Up @@ -393,19 +395,14 @@ def _to_qtimespan(dt):
}



__EPOCH_QDATETIME_MS = int(_EPOCH_QDATETIME.value // 1e6)
__MILIS_PER_DAY_FLOAT = float(_MILIS_PER_DAY)
__EPOCH_QTIMESTAMP_NS = _EPOCH_TIMESTAMP.value

_TO_RAW_LIST = {
QMONTH: lambda a: (a - 360).astype(numpy.int32),
QDATE: lambda a: (a - 10957).astype(numpy.int32),
QDATETIME: lambda a: ((a - __EPOCH_QDATETIME_MS) / __MILIS_PER_DAY_FLOAT).astype(numpy.float64),
QDATETIME: lambda a: ((a - _QEPOCH_MS) / _MILLIS_PER_DAY_FLOAT).astype(numpy.float64),
QMINUTE: lambda a: a.astype(numpy.int32),
QSECOND: lambda a: a.astype(numpy.int32),
QTIME: lambda a: a.astype(numpy.int32),
QTIMESTAMP: lambda a: a - __EPOCH_QTIMESTAMP_NS,
QTIMESTAMP: lambda a: a - _EPOCH_QTIMESTAMP_NS,
QTIMESPAN: None,
}

Expand All @@ -414,11 +411,11 @@ def _to_qtimespan(dt):
_FROM_RAW_LIST = {
QMONTH: lambda a: numpy.array((a + 360), dtype = 'datetime64[M]'),
QDATE: lambda a: numpy.array((a + 10957), dtype = 'datetime64[D]'),
QDATETIME: lambda a: numpy.array((a * _MILIS_PER_DAY + __EPOCH_QDATETIME_MS), dtype = 'datetime64[ms]'),
QDATETIME: lambda a: numpy.array((a * _MILLIS_PER_DAY + _QEPOCH_MS), dtype = 'datetime64[ms]'),
QMINUTE: lambda a: numpy.array(a, dtype = 'timedelta64[m]'),
QSECOND: lambda a: numpy.array(a, dtype = 'timedelta64[s]'),
QTIME: lambda a: numpy.array(a, dtype = 'timedelta64[ms]'),
QTIMESTAMP: lambda a: numpy.array((a + __EPOCH_QTIMESTAMP_NS), dtype = 'datetime64[ns]'),
QTIMESTAMP: lambda a: numpy.array((a + _EPOCH_QTIMESTAMP_NS), dtype = 'datetime64[ns]'),
QTIMESPAN: lambda a: numpy.array(a, dtype = 'timedelta64[ns]'),
}

Expand Down
4 changes: 2 additions & 2 deletions tests/pandas_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,15 +76,15 @@
'meta': MetaData(qtype = QMONTH_LIST) }),
('2001.01.01 2000.05.01 0Nd', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')),
'meta': MetaData(qtype = QDATE_LIST) }),
('2000.01.04T05:36:57.600 0Nz', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')])),
('2000.01.04T05:36:57.600 0Nz', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')])),
'meta': MetaData(qtype = QDATETIME_LIST) }),
('12:01 0Nu', {'data': pandas.Series(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')])),
'meta': MetaData(qtype = QMINUTE_LIST) }),
('12:05:00 0Nv', {'data': pandas.Series(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')])),
'meta': MetaData(qtype = QSECOND_LIST) }),
('12:04:59.123 0Nt', {'data': pandas.Series(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')])),
'meta': MetaData(qtype = QTIME_LIST) }),
('2000.01.04D05:36:57.600 0Np', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), numpy.datetime64('nat', 'ns')])),
('2000.01.04D05:36:57.600 0Np', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')])),
'meta': MetaData(qtype = QTIMESTAMP_LIST) }),
('0D05:36:57.600 0Nn', {'data': pandas.Series(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')])),
'meta': MetaData(qtype = QTIMESPAN_LIST) }),
Expand Down
12 changes: 6 additions & 6 deletions tests/qreader_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
('2001.01.01', qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE)),
('0Nd', qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE)),
('2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST)),
('2000.01.04T05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), qtype=QDATETIME)),
('2000.01.04T05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), qtype=QDATETIME)),
('0Nz', qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME)),
('12:01 0Nu', qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST)),
('12:01', qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE)),
Expand All @@ -51,7 +51,7 @@
('12:04:59.123', qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME)),
('0Nt', qtemporal(numpy.timedelta64('NaT', 'ms'), qtype=QTIME)),
('2000.01.04D05:36:57.600 0Np', qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST)),
('2000.01.04D05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), qtype=QTIMESTAMP)),
('2000.01.04D05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), qtype=QTIMESTAMP)),
('0Np', qtemporal(numpy.datetime64('NaT', 'ns'), qtype=QTIMESTAMP)),
('0D05:36:57.600 0Nn', qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST)),
('0D05:36:57.600', qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN)),
Expand Down Expand Up @@ -179,8 +179,8 @@
('2001.01.01 2000.05.01 0Nd', qlist(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), qtype=QDATE_LIST)),
('2001.01.01', numpy.datetime64('2001-01-01', 'D')),
('0Nd', numpy.datetime64('NaT', 'D')),
('2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST)),
('2000.01.04T05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600', 'ms')),
('2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST)),
('2000.01.04T05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms')),
('0Nz', numpy.datetime64('NaT', 'ms')),
('12:01 0Nu', qlist(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype = QMINUTE)),
('12:01', numpy.timedelta64(721, 'm')),
Expand All @@ -191,8 +191,8 @@
('12:04:59.123 0Nt', qlist(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]), qtype = QTIME_LIST)),
('12:04:59.123', numpy.timedelta64(43499123, 'ms')),
('0Nt', numpy.timedelta64('NaT', 'ms')),
('2000.01.04D05:36:57.600 0Np', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST)),
('2000.01.04D05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600', 'ns')),
('2000.01.04D05:36:57.600 0Np', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST)),
('2000.01.04D05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns')),
('0Np', numpy.datetime64('NaT', 'ns')),
('0D05:36:57.600 0Nn', qlist(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype = QTIMESPAN_LIST)),
('0D05:36:57.600', numpy.timedelta64(20217600000000, 'ns')),
Expand Down
12 changes: 6 additions & 6 deletions tests/qtypes_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import * # @UnusedWildImport
from qpython.qtemporal import * # @UnusedWildImport
from qpython.qtemporal import _MILIS_PER_DAY
from qpython.qtemporal import _MILLIS_PER_DAY


def test_is_null():
Expand Down Expand Up @@ -191,11 +191,11 @@ def test_array_to_raw_qtemporal():
assert na[x] == x - 365
x += 1

na_dt = numpy.arange('1999-01-01T00:00:00.000', '2001-01-04T05:36:57.600', 12345678, dtype='datetime64[ms]')
na_dt = numpy.arange('1999-01-01T00:00:00.000Z', '2001-01-04T05:36:57.600Z', 12345678, dtype='datetime64[ms]')
na = array_to_raw_qtemporal(na_dt, qtype = QDATETIME_LIST)
assert na.dtype == numpy.float64

step = 12346678. / _MILIS_PER_DAY
step = 12346678. / _MILLIS_PER_DAY

assert na[0] == -365.0
assert abs(na[-1] - 369.1677) < 0.001
Expand All @@ -204,7 +204,7 @@ def test_array_to_raw_qtemporal():
ref = (x * step) - 365
assert abs(na[x] - ref) < 0.1, '%s %s' %(na[x], ref)

na_dt = numpy.arange('1999-01-01T00:00:00.000', '2001-01-04T05:36:57.600', 1234567890000, dtype='datetime64[ns]')
na_dt = numpy.arange('1999-01-01T00:00:00.000Z', '2001-01-04T05:36:57.600Z', 1234567890000, dtype='datetime64[ns]')
na = array_to_raw_qtemporal(na_dt, qtype = QTIMESTAMP_LIST)
assert na.dtype == numpy.int64

Expand Down Expand Up @@ -306,14 +306,14 @@ def test_array_from_raw_qtemporal():
assert str(na_dt.dtype).startswith('datetime64[ns]')
for x in xrange(len(na_dt)):
if na_dt[x] != numpy.datetime64('NaT', 'ns'):
assert na_dt[x].astype(numpy.int64) == raw[x] + numpy.datetime64('2000-01-01T00:00:00', 'ns').astype(long)
assert na_dt[x].astype(numpy.int64) == raw[x] + numpy.datetime64('2000-01-01T00:00:00Z', 'ns').astype(long)
else:
assert raw[x] == qnull(QTIMESTAMP)


raw = numpy.array([3.234, qnull(QDATETIME)])
na_dt = array_from_raw_qtemporal(raw, qtype = QDATETIME)
ref = numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')])
ref = numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')])

assert str(na_dt.dtype).startswith('datetime64[ms]')
for x in xrange(len(na_dt)):
Expand Down
24 changes: 12 additions & 12 deletions tests/qwriter_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,13 @@
qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE),
numpy.datetime64('NaT', 'D'))),
('2000.01.04T05:36:57.600 0Nz', (qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')]), qtype=QDATETIME_LIST), qtype=QDATETIME_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]), qtype=QDATETIME_LIST), qtype=QDATETIME_LIST),
qlist([3.234, qnull(QDATETIME)], qtype=QDATETIME_LIST),
qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST),
numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')])
qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST),
numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')])
)),
('2000.01.04T05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), qtype=QDATETIME),
numpy.datetime64('2000-01-04T05:36:57.600', 'ms'))),
('2000.01.04T05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), qtype=QDATETIME),
numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'))),
('0Nz', (qtemporal(qnull(QDATETIME), qtype=QDATETIME),
qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME),
numpy.datetime64('NaT', 'ms'))),
Expand Down Expand Up @@ -98,13 +98,13 @@
qtemporal(numpy.timedelta64('NaT', 'ms'), qtype=QTIME),
numpy.timedelta64('NaT', 'ms'))),
('2000.01.04D05:36:57.600 0Np', (qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), numpy.datetime64('nat', 'ns')]), qtype=QTIMESTAMP_LIST), qtype=QTIMESTAMP_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]), qtype=QTIMESTAMP_LIST), qtype=QTIMESTAMP_LIST),
qlist([long(279417600000000), qnull(QTIMESTAMP)], qtype=QTIMESTAMP_LIST),
qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST),
numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), numpy.datetime64('nat', 'ns')])
qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST),
numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')])
)),
('2000.01.04D05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), qtype=QTIMESTAMP),
numpy.datetime64('2000-01-04T05:36:57.600', 'ns'))),
('2000.01.04D05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), qtype=QTIMESTAMP),
numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'))),
('0Np', (qtemporal(qnull(QTIMESTAMP), qtype=QTIMESTAMP),
qtemporal(numpy.datetime64('NaT', 'ns'), qtype=QTIMESTAMP),
numpy.datetime64('NaT', 'ns'))),
Expand Down Expand Up @@ -339,11 +339,11 @@ def test_writing():
for object in value:
sys.stdout.write( '.' )
serialized = binascii.hexlify(w.write(object, 1))[16:].lower()
assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (object, BINARY[query].lower(), serialized)
assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized)
else:
sys.stdout.write( '.' )
serialized = binascii.hexlify(w.write(value, 1))[16:].lower()
assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (value, BINARY[query].lower(), serialized)
assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized)

print ''

Expand Down

0 comments on commit cce25b6

Please sign in to comment.