adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1454 from freqtrade/feat/interpolate_missing

Browse Source 
interpolate missing candles
This commit is contained in:
Misagh
2019-01-04 22:33:53 +01:00
committed by GitHub
parent 55235ce20d 672d115eca
commit 26a77e193e
13 changed files with 157 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
freqtrade/tests/conftest.py
View File

@@ -542,7 +542,7 @@ def ticker_history_list():
@pytest.fixture
def ticker_history(ticker_history_list):
return parse_ticker_dataframe(ticker_history_list)
return parse_ticker_dataframe(ticker_history_list, "5m", True)
@pytest.fixture
@@ -724,7 +724,7 @@ def tickers():
@pytest.fixture
def result():
with open('freqtrade/tests/testdata/UNITTEST_BTC-1m.json') as data_file:
return parse_ticker_dataframe(json.load(data_file))
return parse_ticker_dataframe(json.load(data_file), '1m', True)
# FIX:
# Create an fixture/function

94
freqtrade/tests/data/test_converter.py
View File

@@ -1,25 +1,99 @@
# pragma pylint: disable=missing-docstring, C0103
import logging
from freqtrade.data.converter import parse_ticker_dataframe
from freqtrade.data.converter import parse_ticker_dataframe, ohlcv_fill_up_missing_data
from freqtrade.data.history import load_pair_history
from freqtrade.optimize import validate_backtest_data, get_timeframe
from freqtrade.tests.conftest import log_has
def test_dataframe_correct_length(result):
dataframe = parse_ticker_dataframe(result)
assert len(result.index) - 1 == len(dataframe.index) # last partial candle removed
def test_dataframe_correct_columns(result):
assert result.columns.tolist() == \
['date', 'open', 'high', 'low', 'close', 'volume']
assert result.columns.tolist() == ['date', 'open', 'high', 'low', 'close', 'volume']
def test_parse_ticker_dataframe(ticker_history, caplog):
def test_parse_ticker_dataframe(ticker_history_list, caplog):
columns = ['date', 'open', 'high', 'low', 'close', 'volume']
caplog.set_level(logging.DEBUG)
# Test file with BV data
dataframe = parse_ticker_dataframe(ticker_history)
dataframe = parse_ticker_dataframe(ticker_history_list, '5m', fill_missing=True)
assert dataframe.columns.tolist() == columns
assert log_has('Parsing tickerlist to dataframe', caplog.record_tuples)
def test_ohlcv_fill_up_missing_data(caplog):
data = load_pair_history(datadir=None,
ticker_interval='1m',
refresh_pairs=False,
pair='UNITTEST/BTC',
fill_up_missing=False)
caplog.set_level(logging.DEBUG)
data2 = ohlcv_fill_up_missing_data(data, '1m')
assert len(data2) > len(data)
# Column names should not change
assert (data.columns == data2.columns).all()
assert log_has(f"Missing data fillup: before: {len(data)} - after: {len(data2)}",
caplog.record_tuples)
# Test fillup actually fixes invalid backtest data
min_date, max_date = get_timeframe({'UNITTEST/BTC': data})
assert validate_backtest_data({'UNITTEST/BTC': data}, min_date, max_date, 1)
assert not validate_backtest_data({'UNITTEST/BTC': data2}, min_date, max_date, 1)
def test_ohlcv_fill_up_missing_data2(caplog):
ticker_interval = '5m'
ticks = [[
1511686200000, # 8:50:00
8.794e-05, # open
8.948e-05, # high
8.794e-05, # low
8.88e-05, # close
2255, # volume (in quote currency)
],
[
1511686500000, # 8:55:00
8.88e-05,
8.942e-05,
8.88e-05,
8.893e-05,
9911,
],
[
1511687100000, # 9:05:00
8.891e-05,
8.893e-05,
8.875e-05,
8.877e-05,
2251
],
[
1511687400000, # 9:10:00
8.877e-05,
8.883e-05,
8.895e-05,
8.817e-05,
123551
]
]
# Generate test-data without filling missing
data = parse_ticker_dataframe(ticks, ticker_interval, fill_missing=False)
assert len(data) == 3
caplog.set_level(logging.DEBUG)
data2 = ohlcv_fill_up_missing_data(data, ticker_interval)
assert len(data2) == 4
# 3rd candle has been filled
row = data2.loc[2, :]
assert row['volume'] == 0
# close shoult match close of previous candle
assert row['close'] == data.loc[1, 'close']
assert row['open'] == row['close']
assert row['high'] == row['close']
assert row['low'] == row['close']
# Column names should not change
assert (data.columns == data2.columns).all()
assert log_has(f"Missing data fillup: before: {len(data)} - after: {len(data2)}",
caplog.record_tuples)

4
freqtrade/tests/edge/test_edge.py
View File

@@ -281,8 +281,8 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals
123.45
] for x in range(0, 500)]
pairdata = {'NEO/BTC': parse_ticker_dataframe(ETHBTC),
'LTC/BTC': parse_ticker_dataframe(LTCBTC)}
pairdata = {'NEO/BTC': parse_ticker_dataframe(ETHBTC, '1h', fill_missing=True),
'LTC/BTC': parse_ticker_dataframe(LTCBTC, '1h', fill_missing=True)}
return pairdata

12
freqtrade/tests/optimize/test_backtesting.py
View File

@@ -75,7 +75,7 @@ def load_data_test(what):
pair[x][5] # Keep old volume
] for x in range(0, datalen)
]
return {'UNITTEST/BTC': parse_ticker_dataframe(data)}
return {'UNITTEST/BTC': parse_ticker_dataframe(data, '1m', fill_missing=True)}
def simple_backtest(config, contour, num_results, mocker) -> None:
@@ -104,7 +104,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None:
def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False,
timerange=None, exchange=None):
tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange)
pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata)}
pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', fill_missing=True)}
return pairdata
@@ -323,15 +323,15 @@ def test_backtesting_init(mocker, default_conf) -> None:
assert backtesting.fee == 0.5
def test_tickerdata_to_dataframe(default_conf, mocker) -> None:
def test_tickerdata_to_dataframe_bt(default_conf, mocker) -> None:
patch_exchange(mocker)
timerange = TimeRange(None, 'line', 0, -100)
tick = history.load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange)
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)}
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)}
backtesting = Backtesting(default_conf)
data = backtesting.strategy.tickerdata_to_dataframe(tickerlist)
assert len(data['UNITTEST/BTC']) == 99
assert len(data['UNITTEST/BTC']) == 102
# Load strategy to compare the result between Backtesting function and strategy are the same
strategy = DefaultStrategy(default_conf)
@@ -594,7 +594,7 @@ def test_processed(default_conf, mocker) -> None:
def test_backtest_pricecontours(default_conf, fee, mocker) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
tests = [['raise', 18], ['lower', 0], ['sine', 19]]
tests = [['raise', 19], ['lower', 0], ['sine', 18]]
# We need to enable sell-signal - otherwise it sells on ROI!!
default_conf['experimental'] = {"use_sell_signal": True}

4
freqtrade/tests/optimize/test_hyperopt.py
View File

@@ -243,7 +243,7 @@ def test_has_space(hyperopt):
def test_populate_indicators(hyperopt) -> None:
tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m')
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)}
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)}
dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist)
dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'],
{'pair': 'UNITTEST/BTC'})
@@ -256,7 +256,7 @@ def test_populate_indicators(hyperopt) -> None:
def test_buy_strategy_generator(hyperopt) -> None:
tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m')
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)}
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)}
dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist)
dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'],
{'pair': 'UNITTEST/BTC'})

3
freqtrade/tests/optimize/test_optimize.py
View File

@@ -30,7 +30,8 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None:
history.load_data(
datadir=None,
ticker_interval='1m',
pairs=['UNITTEST/BTC']
pairs=['UNITTEST/BTC'],
fill_up_missing=False
)
)
min_date, max_date = optimize.get_timeframe(data)

2
freqtrade/tests/strategy/test_default_strategy.py
View File

@@ -10,7 +10,7 @@ from freqtrade.strategy.default_strategy import DefaultStrategy
@pytest.fixture
def result():
with open('freqtrade/tests/testdata/ETH_BTC-1m.json') as data_file:
return parse_ticker_dataframe(json.load(data_file))
return parse_ticker_dataframe(json.load(data_file), '1m', fill_missing=True)
def test_default_strategy_structure():

4
freqtrade/tests/strategy/test_interface.py
View File

@@ -111,9 +111,9 @@ def test_tickerdata_to_dataframe(default_conf) -> None:
timerange = TimeRange(None, 'line', 0, -100)
tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange)
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)}
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', True)}
data = strategy.tickerdata_to_dataframe(tickerlist)
assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed
assert len(data['UNITTEST/BTC']) == 102 # partial candle was removed
def test_min_roi_reached(default_conf, fee) -> None:

4
freqtrade/tests/test_misc.py
View File

@@ -17,7 +17,7 @@ def test_shorten_date() -> None:
def test_datesarray_to_datetimearray(ticker_history_list):
dataframes = parse_ticker_dataframe(ticker_history_list)
dataframes = parse_ticker_dataframe(ticker_history_list, "5m", fill_missing=True)
dates = datesarray_to_datetimearray(dataframes['date'])
assert isinstance(dates[0], datetime.datetime)
@@ -34,7 +34,7 @@ def test_datesarray_to_datetimearray(ticker_history_list):
def test_common_datearray(default_conf) -> None:
strategy = DefaultStrategy(default_conf)
tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m')
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)}
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, "1m", fill_missing=True)}
dataframes = strategy.tickerdata_to_dataframe(tickerlist)
dates = common_datearray(dataframes)