diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index 3366c3965..b88a44836 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -718,9 +718,12 @@ def save_asset_data(folder, df, decimals=8): def forward_fill_df_if_needed(df, periods): df = df.reindex(periods) - df['volume'] = df['volume'].fillna(0.0)# volume should always be 0 (if there were no trades in this interval) - df['close'] = df.fillna(method='pad') # ie pull the last close into this close - # now copy the close that was pulled down from the last timestep into this row, across into o/h/l + # volume should always be 0 (if there were no trades in this interval) + df['volume'] = df['volume'].fillna(0.0) + # ie pull the last close into this close + df['close'] = df.fillna(method='pad') + # now copy the close that was pulled down from the last timestep + # into this row, across into o/h/l df['open'] = df['open'].fillna(df['close']) df['low'] = df['low'].fillna(df['close']) df['high'] = df['high'].fillna(df['close']) diff --git a/tests/exchange/test_exchange_utils.py b/tests/exchange/test_exchange_utils.py index deebc9f15..ae5c07dc6 100644 --- a/tests/exchange/test_exchange_utils.py +++ b/tests/exchange/test_exchange_utils.py @@ -1,7 +1,8 @@ -from catalyst.exchange.utils.exchange_utils import transform_candles_to_df, forward_fill_df_if_needed, get_candles_df +from catalyst.exchange.utils.exchange_utils import transform_candles_to_df, \ + forward_fill_df_if_needed, get_candles_df from catalyst.testing.fixtures import WithLogger, ZiplineTestCase -from pandas import Timestamp, Series, DataFrame +from pandas import Timestamp, DataFrame import numpy as np @@ -19,52 +20,76 @@ def test_transform_candles_to_series(self): candles = [{'high': 595, 'volume': 10, 'low': 594, 'close': 595, 'open': 594, - 'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')}, + 'last_traded': Timestamp('2018-03-01 09:45:00+0000', + tz='UTC') + }, {'high': 594, 'volume': 108, 'low': 592, 'close': 593, 'open': 592, - 'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')}] + 'last_traded': Timestamp('2018-03-01 09:50:00+0000', + tz='UTC') + }] expected = [{'high': 595.0, 'volume': 10.0, 'low': 594.0, - 'close': 595.0, 'open': 594.0, - 'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')}, - {'high': 594.0, 'volume': 108.0, 'low': 592.0, - 'close': 593.0, 'open': 592.0, - 'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')}, - {'high': 593.0, 'volume': 0.0, 'low': 593.0, - 'close': 593.0, 'open': 593.0, - 'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')} - ] + 'close': 595.0, 'open': 594.0, + 'last_traded': Timestamp('2018-03-01 09:45:00+0000', + tz='UTC') + }, + {'high': 594.0, 'volume': 108.0, 'low': 592.0, + 'close': 593.0, 'open': 592.0, + 'last_traded': Timestamp('2018-03-01 09:50:00+0000', + tz='UTC') + }, + {'high': 593.0, 'volume': 0.0, 'low': 593.0, + 'close': 593.0, 'open': 593.0, + 'last_traded': Timestamp('2018-03-01 09:55:00+0000', + tz='UTC') + }] periods = [Timestamp('2018-03-01 09:45:00+0000', tz='UTC'), Timestamp('2018-03-01 09:50:00+0000', tz='UTC'), Timestamp('2018-03-01 09:55:00+0000', tz='UTC')] - observed_df = forward_fill_df_if_needed(transform_candles_to_df(candles), periods) + observed_df = forward_fill_df_if_needed( + transform_candles_to_df(candles), + periods) expected_df = transform_candles_to_df(expected) assert (expected_df.equals(observed_df)) for field in ['volume', 'open', 'close', 'high', 'low']: - assert(self.get_specific_field_from_df(observed_df, field, asset).equals( - get_candles_df({asset:candles}, field, '5T', 3, end_dt=periods[2]))) + field_dt = self.get_specific_field_from_df(observed_df, + field, + asset) + assert (field_dt.equals(get_candles_df({asset: candles}, + field, '5T', 3, + end_dt=periods[2]))) candles = [{'high': 595, 'volume': 10, 'low': 594, 'close': 595, 'open': 594, - 'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')}, + 'last_traded': Timestamp('2018-03-01 09:45:00+0000', + tz='UTC') + }, {'high': 594, 'volume': 108, 'low': 592, 'close': 593, 'open': 592, - 'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')}] + 'last_traded': Timestamp('2018-03-01 09:55:00+0000', + tz='UTC') + }] expected = [{'high': 595.0, 'volume': 10.0, 'low': 594.0, 'close': 595.0, 'open': 594.0, - 'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')}, + 'last_traded': Timestamp('2018-03-01 09:45:00+0000', + tz='UTC') + }, {'high': 595.0, 'volume': 0.0, 'low': 595.0, 'close': 595.0, 'open': 595.0, - 'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')}, + 'last_traded': Timestamp('2018-03-01 09:50:00+0000', + tz='UTC') + }, {'high': 594.0, 'volume': 108.0, 'low': 592.0, 'close': 593.0, 'open': 592.0, - 'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')} - ] + 'last_traded': Timestamp('2018-03-01 09:55:00+0000', + tz='UTC') + }] df = transform_candles_to_df(candles) observed_df = forward_fill_df_if_needed(df, periods) @@ -72,26 +97,39 @@ def test_transform_candles_to_series(self): assert (transform_candles_to_df(expected).equals(observed_df)) for field in ['volume', 'open', 'close', 'high', 'low']: - assert(self.get_specific_field_from_df(observed_df, field, asset).equals( - get_candles_df({asset:candles}, field, '5T', 3, end_dt=periods[2]))) + field_dt = self.get_specific_field_from_df(observed_df, + field, + asset) + assert(field_dt.equals(get_candles_df({asset: candles}, + field, '5T', 3, + end_dt=periods[2]))) candles = [{'high': 595, 'volume': 10, 'low': 594, 'close': 595, 'open': 594, - 'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')}, + 'last_traded': Timestamp('2018-03-01 09:50:00+0000', + tz='UTC') + }, {'high': 594, 'volume': 108, 'low': 592, 'close': 593, 'open': 592, - 'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')}] + 'last_traded': Timestamp('2018-03-01 09:55:00+0000', + tz='UTC') + }] expected = [{'high': np.NaN, 'volume': 0.0, 'low': np.NaN, 'close': np.NaN, 'open': np.NaN, - 'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')}, + 'last_traded': Timestamp('2018-03-01 09:45:00+0000', + tz='UTC') + }, {'high': 595, 'volume': 10, 'low': 594, 'close': 595, 'open': 594, - 'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')}, + 'last_traded': Timestamp('2018-03-01 09:50:00+0000', + tz='UTC') + }, {'high': 594, 'volume': 108, 'low': 592, 'close': 593, 'open': 592, - 'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')} - ] + 'last_traded': Timestamp('2018-03-01 09:55:00+0000', + tz='UTC') + }] df = transform_candles_to_df(candles) observed_df = forward_fill_df_if_needed(df, periods) @@ -99,7 +137,11 @@ def test_transform_candles_to_series(self): assert (transform_candles_to_df(expected).equals(observed_df)) # Not the same due to dropna - commenting out for now """ - for field in ['volume', 'open', 'close', 'high', 'low']: - assert(self.get_specific_field_from_df(observed_df, field, asset).equals( - get_candles_df({asset:candles}, field, '5T', 3, end_dt=periods[2]))) - """ \ No newline at end of file + for field in ['volume', 'open', 'close', 'high', 'low']: + field_dt = self.get_specific_field_from_df(observed_df, + field, + asset) + assert(field_dt.equals(get_candles_df({asset:candles}, + field, '5T', 3, + end_dt=periods[2]))) + """