Implement missing_data_fillup to tests and operations

freqtrade/exchange/__init__.py

@@ -558,7 +558,7 @@ class Exchange(object):
             if ticks:
                 self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000
             # keeping parsed dataframe in cache
-            self._klines[pair] = parse_ticker_dataframe(ticks)
+            self._klines[pair] = parse_ticker_dataframe(ticks, tick_interval, fill_missing=True)
         return tickers
 
     @retrier_async

freqtrade/optimize/backtesting.py

@@ -19,7 +19,6 @@ from freqtrade.arguments import Arguments
 from freqtrade.configuration import Configuration
 from freqtrade.exchange import Exchange
 from freqtrade.data import history
-from freqtrade.data.converter import ohlcv_fill_up_missing_data
 from freqtrade.misc import file_dump_json
 from freqtrade.persistence import Trade
 from freqtrade.resolvers import StrategyResolver
@@ -395,7 +394,7 @@ class Backtesting(object):
             self._set_strategy(strat)
 
             min_date, max_date = optimize.get_timeframe(data)
-            # Validate dataframe for missing values
+            # Validate dataframe for missing values (mainly at start and end, as fillup is called)
             optimize.validate_backtest_data(data, min_date, max_date,
                                             constants.TICKER_INTERVAL_MINUTES[self.ticker_interval])
             logger.info(

freqtrade/tests/conftest.py

@@ -542,7 +542,7 @@ def ticker_history_list():
 
 @pytest.fixture
 def ticker_history(ticker_history_list):
-    return parse_ticker_dataframe(ticker_history_list)
+    return parse_ticker_dataframe(ticker_history_list, "5m", True)
 
 
 @pytest.fixture
@@ -724,7 +724,7 @@ def tickers():
 @pytest.fixture
 def result():
     with open('freqtrade/tests/testdata/UNITTEST_BTC-1m.json') as data_file:
-        return parse_ticker_dataframe(json.load(data_file))
+        return parse_ticker_dataframe(json.load(data_file), '1m', True)
 
 # FIX:
 # Create an fixture/function

freqtrade/tests/data/test_converter.py

@@ -7,21 +7,16 @@ from freqtrade.optimize import validate_backtest_data, get_timeframe
 from freqtrade.tests.conftest import log_has
 
 
-def test_dataframe_correct_length(result):
-    dataframe = parse_ticker_dataframe(result)
-    assert len(result.index) - 1 == len(dataframe.index)    # last partial candle removed
-
-
 def test_dataframe_correct_columns(result):
     assert result.columns.tolist() == ['date', 'open', 'high', 'low', 'close', 'volume']
 
 
-def test_parse_ticker_dataframe(ticker_history, caplog):
+def test_parse_ticker_dataframe(ticker_history_list, caplog):
     columns = ['date', 'open', 'high', 'low', 'close', 'volume']
 
     caplog.set_level(logging.DEBUG)
     # Test file with BV data
-    dataframe = parse_ticker_dataframe(ticker_history)
+    dataframe = parse_ticker_dataframe(ticker_history_list, '5m', fill_missing=True)
     assert dataframe.columns.tolist() == columns
     assert log_has('Parsing tickerlist to dataframe', caplog.record_tuples)
 
@@ -30,7 +25,8 @@ def test_ohlcv_fill_up_missing_data(caplog):
     data = load_pair_history(datadir=None,
                              ticker_interval='1m',
                              refresh_pairs=False,
-                             pair='UNITTEST/BTC')
+                             pair='UNITTEST/BTC',
+                             fill_up_missing=False)
     caplog.set_level(logging.DEBUG)
     data2 = ohlcv_fill_up_missing_data(data, '1m')
     assert len(data2) > len(data)

freqtrade/tests/edge/test_edge.py

@@ -281,8 +281,8 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals
             123.45
         ] for x in range(0, 500)]
 
-    pairdata = {'NEO/BTC': parse_ticker_dataframe(ETHBTC),
+    pairdata = {'NEO/BTC': parse_ticker_dataframe(ETHBTC, '1h', fill_missing=True),
-                'LTC/BTC': parse_ticker_dataframe(LTCBTC)}
+                'LTC/BTC': parse_ticker_dataframe(LTCBTC, '1h', fill_missing=True)}
     return pairdata
 
 

freqtrade/tests/optimize/test_backtesting.py

@@ -75,7 +75,7 @@ def load_data_test(what):
                 pair[x][5]  # Keep old volume
             ] for x in range(0, datalen)
         ]
-    return {'UNITTEST/BTC': parse_ticker_dataframe(data)}
+    return {'UNITTEST/BTC': parse_ticker_dataframe(data, '1m', fill_missing=True)}
 
 
 def simple_backtest(config, contour, num_results, mocker) -> None:
@@ -104,7 +104,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None:
 def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False,
                      timerange=None, exchange=None):
     tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange)
-    pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata)}
+    pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', fill_missing=True)}
     return pairdata
 
 
@@ -322,15 +322,15 @@ def test_backtesting_init(mocker, default_conf) -> None:
     assert backtesting.fee == 0.5
 
 
-def test_tickerdata_to_dataframe(default_conf, mocker) -> None:
+def test_tickerdata_to_dataframe_bt(default_conf, mocker) -> None:
     patch_exchange(mocker)
     timerange = TimeRange(None, 'line', 0, -100)
     tick = history.load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange)
-    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)}
+    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)}
 
     backtesting = Backtesting(default_conf)
     data = backtesting.strategy.tickerdata_to_dataframe(tickerlist)
-    assert len(data['UNITTEST/BTC']) == 99
+    assert len(data['UNITTEST/BTC']) == 102
 
     # Load strategy to compare the result between Backtesting function and strategy are the same
     strategy = DefaultStrategy(default_conf)
@@ -593,7 +593,7 @@ def test_processed(default_conf, mocker) -> None:
 
 def test_backtest_pricecontours(default_conf, fee, mocker) -> None:
     mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
-    tests = [['raise', 18], ['lower', 0], ['sine', 19]]
+    tests = [['raise', 19], ['lower', 0], ['sine', 18]]
     # We need to enable sell-signal - otherwise it sells on ROI!!
     default_conf['experimental'] = {"use_sell_signal": True}
 

freqtrade/tests/optimize/test_hyperopt.py

@@ -243,7 +243,7 @@ def test_has_space(hyperopt):
 
 def test_populate_indicators(hyperopt) -> None:
     tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m')
-    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)}
+    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)}
     dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist)
     dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'],
                                                              {'pair': 'UNITTEST/BTC'})
@@ -256,7 +256,7 @@ def test_populate_indicators(hyperopt) -> None:
 
 def test_buy_strategy_generator(hyperopt) -> None:
     tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m')
-    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)}
+    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)}
     dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist)
     dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'],
                                                              {'pair': 'UNITTEST/BTC'})

freqtrade/tests/optimize/test_optimize.py

@@ -30,7 +30,8 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None:
         history.load_data(
             datadir=None,
             ticker_interval='1m',
-            pairs=['UNITTEST/BTC']
+            pairs=['UNITTEST/BTC'],
+            fill_up_missing=False
         )
     )
     min_date, max_date = optimize.get_timeframe(data)

freqtrade/tests/strategy/test_default_strategy.py

@@ -10,7 +10,7 @@ from freqtrade.strategy.default_strategy import DefaultStrategy
 @pytest.fixture
 def result():
     with open('freqtrade/tests/testdata/ETH_BTC-1m.json') as data_file:
-        return parse_ticker_dataframe(json.load(data_file))
+        return parse_ticker_dataframe(json.load(data_file), '1m', fill_missing=True)
 
 
 def test_default_strategy_structure():

freqtrade/tests/strategy/test_interface.py

@@ -111,9 +111,9 @@ def test_tickerdata_to_dataframe(default_conf) -> None:
 
     timerange = TimeRange(None, 'line', 0, -100)
     tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange)
-    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)}
+    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', True)}
     data = strategy.tickerdata_to_dataframe(tickerlist)
-    assert len(data['UNITTEST/BTC']) == 99       # partial candle was removed
+    assert len(data['UNITTEST/BTC']) == 102       # partial candle was removed
 
 
 def test_min_roi_reached(default_conf, fee) -> None:

freqtrade/tests/test_misc.py

@@ -17,7 +17,7 @@ def test_shorten_date() -> None:
 
 
 def test_datesarray_to_datetimearray(ticker_history_list):
-    dataframes = parse_ticker_dataframe(ticker_history_list)
+    dataframes = parse_ticker_dataframe(ticker_history_list, "5m", fill_missing=True)
     dates = datesarray_to_datetimearray(dataframes['date'])
 
     assert isinstance(dates[0], datetime.datetime)
@@ -34,7 +34,7 @@ def test_datesarray_to_datetimearray(ticker_history_list):
 def test_common_datearray(default_conf) -> None:
     strategy = DefaultStrategy(default_conf)
     tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m')
-    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)}
+    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, "1m", fill_missing=True)}
     dataframes = strategy.tickerdata_to_dataframe(tickerlist)
 
     dates = common_datearray(dataframes)