Skip to content

Commit

Permalink
BLD:flake fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@lenak25
lenak25 committed Mar 1, 2018
1 parent b95cf46 commit 868f17f
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 37 deletions.
9 changes: 6 additions & 3 deletions catalyst/exchange/utils/exchange_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -718,9 +718,12 @@ def save_asset_data(folder, df, decimals=8):

def forward_fill_df_if_needed(df, periods):
df = df.reindex(periods)
df['volume'] = df['volume'].fillna(0.0)# volume should always be 0 (if there were no trades in this interval)
df['close'] = df.fillna(method='pad') # ie pull the last close into this close
# now copy the close that was pulled down from the last timestep into this row, across into o/h/l
# volume should always be 0 (if there were no trades in this interval)
df['volume'] = df['volume'].fillna(0.0)
# ie pull the last close into this close
df['close'] = df.fillna(method='pad')
# now copy the close that was pulled down from the last timestep
# into this row, across into o/h/l
df['open'] = df['open'].fillna(df['close'])
df['low'] = df['low'].fillna(df['close'])
df['high'] = df['high'].fillna(df['close'])
Expand Down
110 changes: 76 additions & 34 deletions tests/exchange/test_exchange_utils.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
from catalyst.exchange.utils.exchange_utils import transform_candles_to_df, forward_fill_df_if_needed, get_candles_df
from catalyst.exchange.utils.exchange_utils import transform_candles_to_df, \
forward_fill_df_if_needed, get_candles_df

from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
from pandas import Timestamp, Series, DataFrame
from pandas import Timestamp, DataFrame

import numpy as np

Expand All @@ -19,87 +20,128 @@ def test_transform_candles_to_series(self):

candles = [{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')},
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')}]
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
}]

expected = [{'high': 595.0, 'volume': 10.0, 'low': 594.0,
'close': 595.0, 'open': 594.0,
'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')},
{'high': 594.0, 'volume': 108.0, 'low': 592.0,
'close': 593.0, 'open': 592.0,
'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')},
{'high': 593.0, 'volume': 0.0, 'low': 593.0,
'close': 593.0, 'open': 593.0,
'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')}
]
'close': 595.0, 'open': 594.0,
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 594.0, 'volume': 108.0, 'low': 592.0,
'close': 593.0, 'open': 592.0,
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
},
{'high': 593.0, 'volume': 0.0, 'low': 593.0,
'close': 593.0, 'open': 593.0,
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]

periods = [Timestamp('2018-03-01 09:45:00+0000', tz='UTC'),
Timestamp('2018-03-01 09:50:00+0000', tz='UTC'),
Timestamp('2018-03-01 09:55:00+0000', tz='UTC')]

observed_df = forward_fill_df_if_needed(transform_candles_to_df(candles), periods)
observed_df = forward_fill_df_if_needed(
transform_candles_to_df(candles),
periods)
expected_df = transform_candles_to_df(expected)

assert (expected_df.equals(observed_df))

for field in ['volume', 'open', 'close', 'high', 'low']:
assert(self.get_specific_field_from_df(observed_df, field, asset).equals(
get_candles_df({asset:candles}, field, '5T', 3, end_dt=periods[2])))
field_dt = self.get_specific_field_from_df(observed_df,
field,
asset)
assert (field_dt.equals(get_candles_df({asset: candles},
field, '5T', 3,
end_dt=periods[2])))

candles = [{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')},
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')}]
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]

expected = [{'high': 595.0, 'volume': 10.0, 'low': 594.0,
'close': 595.0, 'open': 594.0,
'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')},
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 595.0, 'volume': 0.0, 'low': 595.0,
'close': 595.0, 'open': 595.0,
'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')},
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
},
{'high': 594.0, 'volume': 108.0, 'low': 592.0,
'close': 593.0, 'open': 592.0,
'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')}
]
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]

df = transform_candles_to_df(candles)
observed_df = forward_fill_df_if_needed(df, periods)

assert (transform_candles_to_df(expected).equals(observed_df))

for field in ['volume', 'open', 'close', 'high', 'low']:
assert(self.get_specific_field_from_df(observed_df, field, asset).equals(
get_candles_df({asset:candles}, field, '5T', 3, end_dt=periods[2])))
field_dt = self.get_specific_field_from_df(observed_df,
field,
asset)
assert(field_dt.equals(get_candles_df({asset: candles},
field, '5T', 3,
end_dt=periods[2])))

candles = [{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')},
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')}]
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]

expected = [{'high': np.NaN, 'volume': 0.0, 'low': np.NaN,
'close': np.NaN, 'open': np.NaN,
'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')},
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')},
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')}
]
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]

df = transform_candles_to_df(candles)
observed_df = forward_fill_df_if_needed(df, periods)

assert (transform_candles_to_df(expected).equals(observed_df))
# Not the same due to dropna - commenting out for now
"""
for field in ['volume', 'open', 'close', 'high', 'low']:
assert(self.get_specific_field_from_df(observed_df, field, asset).equals(
get_candles_df({asset:candles}, field, '5T', 3, end_dt=periods[2])))
"""
for field in ['volume', 'open', 'close', 'high', 'low']:
field_dt = self.get_specific_field_from_df(observed_df,
field,
asset)
assert(field_dt.equals(get_candles_df({asset:candles},
field, '5T', 3,
end_dt=periods[2])))
"""

0 comments on commit 868f17f

Please sign in to comment.