diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 98522b98a..c74b32ad2 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -558,7 +558,7 @@ class Exchange(object): if ticks: self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache - self._klines[pair] = parse_ticker_dataframe(ticks) + self._klines[pair] = parse_ticker_dataframe(ticks, tick_interval, fill_missing=True) return tickers @retrier_async diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 86b5bdd19..5ebb5e3a6 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -19,7 +19,6 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange from freqtrade.data import history -from freqtrade.data.converter import ohlcv_fill_up_missing_data from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade from freqtrade.resolvers import StrategyResolver @@ -395,7 +394,7 @@ class Backtesting(object): self._set_strategy(strat) min_date, max_date = optimize.get_timeframe(data) - # Validate dataframe for missing values + # Validate dataframe for missing values (mainly at start and end, as fillup is called) optimize.validate_backtest_data(data, min_date, max_date, constants.TICKER_INTERVAL_MINUTES[self.ticker_interval]) logger.info( diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 0a2633c7c..ba2e1e84e 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -542,7 +542,7 @@ def ticker_history_list(): @pytest.fixture def ticker_history(ticker_history_list): - return parse_ticker_dataframe(ticker_history_list) + return parse_ticker_dataframe(ticker_history_list, "5m", True) @pytest.fixture @@ -724,7 +724,7 @@ def tickers(): @pytest.fixture def result(): with open('freqtrade/tests/testdata/UNITTEST_BTC-1m.json') as data_file: - return parse_ticker_dataframe(json.load(data_file)) + return parse_ticker_dataframe(json.load(data_file), '1m', True) # FIX: # Create an fixture/function diff --git a/freqtrade/tests/data/test_converter.py b/freqtrade/tests/data/test_converter.py index aa4a161a7..8d732c70f 100644 --- a/freqtrade/tests/data/test_converter.py +++ b/freqtrade/tests/data/test_converter.py @@ -7,21 +7,16 @@ from freqtrade.optimize import validate_backtest_data, get_timeframe from freqtrade.tests.conftest import log_has -def test_dataframe_correct_length(result): - dataframe = parse_ticker_dataframe(result) - assert len(result.index) - 1 == len(dataframe.index) # last partial candle removed - - def test_dataframe_correct_columns(result): assert result.columns.tolist() == ['date', 'open', 'high', 'low', 'close', 'volume'] -def test_parse_ticker_dataframe(ticker_history, caplog): +def test_parse_ticker_dataframe(ticker_history_list, caplog): columns = ['date', 'open', 'high', 'low', 'close', 'volume'] caplog.set_level(logging.DEBUG) # Test file with BV data - dataframe = parse_ticker_dataframe(ticker_history) + dataframe = parse_ticker_dataframe(ticker_history_list, '5m', fill_missing=True) assert dataframe.columns.tolist() == columns assert log_has('Parsing tickerlist to dataframe', caplog.record_tuples) @@ -30,7 +25,8 @@ def test_ohlcv_fill_up_missing_data(caplog): data = load_pair_history(datadir=None, ticker_interval='1m', refresh_pairs=False, - pair='UNITTEST/BTC') + pair='UNITTEST/BTC', + fill_up_missing=False) caplog.set_level(logging.DEBUG) data2 = ohlcv_fill_up_missing_data(data, '1m') assert len(data2) > len(data) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 4fbe9c494..c1c1b49cd 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -281,8 +281,8 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals 123.45 ] for x in range(0, 500)] - pairdata = {'NEO/BTC': parse_ticker_dataframe(ETHBTC), - 'LTC/BTC': parse_ticker_dataframe(LTCBTC)} + pairdata = {'NEO/BTC': parse_ticker_dataframe(ETHBTC, '1h', fill_missing=True), + 'LTC/BTC': parse_ticker_dataframe(LTCBTC, '1h', fill_missing=True)} return pairdata diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 8f7c80523..db044d8cd 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -75,7 +75,7 @@ def load_data_test(what): pair[x][5] # Keep old volume ] for x in range(0, datalen) ] - return {'UNITTEST/BTC': parse_ticker_dataframe(data)} + return {'UNITTEST/BTC': parse_ticker_dataframe(data, '1m', fill_missing=True)} def simple_backtest(config, contour, num_results, mocker) -> None: @@ -104,7 +104,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None: def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, timerange=None, exchange=None): tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) - pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata)} + pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', fill_missing=True)} return pairdata @@ -322,15 +322,15 @@ def test_backtesting_init(mocker, default_conf) -> None: assert backtesting.fee == 0.5 -def test_tickerdata_to_dataframe(default_conf, mocker) -> None: +def test_tickerdata_to_dataframe_bt(default_conf, mocker) -> None: patch_exchange(mocker) timerange = TimeRange(None, 'line', 0, -100) tick = history.load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)} + tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)} backtesting = Backtesting(default_conf) data = backtesting.strategy.tickerdata_to_dataframe(tickerlist) - assert len(data['UNITTEST/BTC']) == 99 + assert len(data['UNITTEST/BTC']) == 102 # Load strategy to compare the result between Backtesting function and strategy are the same strategy = DefaultStrategy(default_conf) @@ -593,7 +593,7 @@ def test_processed(default_conf, mocker) -> None: def test_backtest_pricecontours(default_conf, fee, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - tests = [['raise', 18], ['lower', 0], ['sine', 19]] + tests = [['raise', 19], ['lower', 0], ['sine', 18]] # We need to enable sell-signal - otherwise it sells on ROI!! default_conf['experimental'] = {"use_sell_signal": True} diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 5a40441bb..53d991c09 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -243,7 +243,7 @@ def test_has_space(hyperopt): def test_populate_indicators(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)} + tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)} dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) @@ -256,7 +256,7 @@ def test_populate_indicators(hyperopt) -> None: def test_buy_strategy_generator(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)} + tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)} dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 02d0e9c36..99cd24c26 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -30,7 +30,8 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: history.load_data( datadir=None, ticker_interval='1m', - pairs=['UNITTEST/BTC'] + pairs=['UNITTEST/BTC'], + fill_up_missing=False ) ) min_date, max_date = optimize.get_timeframe(data) diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index 45ed54c4d..be514f2d1 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -10,7 +10,7 @@ from freqtrade.strategy.default_strategy import DefaultStrategy @pytest.fixture def result(): with open('freqtrade/tests/testdata/ETH_BTC-1m.json') as data_file: - return parse_ticker_dataframe(json.load(data_file)) + return parse_ticker_dataframe(json.load(data_file), '1m', fill_missing=True) def test_default_strategy_structure(): diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 22ba9a2b6..32e6338bd 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -111,9 +111,9 @@ def test_tickerdata_to_dataframe(default_conf) -> None: timerange = TimeRange(None, 'line', 0, -100) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)} + tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', True)} data = strategy.tickerdata_to_dataframe(tickerlist) - assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed + assert len(data['UNITTEST/BTC']) == 102 # partial candle was removed def test_min_roi_reached(default_conf, fee) -> None: diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 33a59effd..c7ff1f077 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -17,7 +17,7 @@ def test_shorten_date() -> None: def test_datesarray_to_datetimearray(ticker_history_list): - dataframes = parse_ticker_dataframe(ticker_history_list) + dataframes = parse_ticker_dataframe(ticker_history_list, "5m", fill_missing=True) dates = datesarray_to_datetimearray(dataframes['date']) assert isinstance(dates[0], datetime.datetime) @@ -34,7 +34,7 @@ def test_datesarray_to_datetimearray(ticker_history_list): def test_common_datearray(default_conf) -> None: strategy = DefaultStrategy(default_conf) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)} + tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, "1m", fill_missing=True)} dataframes = strategy.tickerdata_to_dataframe(tickerlist) dates = common_datearray(dataframes)