Move most logic to history

freqtrade/data/history.py

@@ -8,6 +8,7 @@ Includes:
 
 import logging
 import operator
+from copy import deepcopy
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
@@ -19,7 +20,7 @@ from pandas import DataFrame
 from freqtrade import OperationalException, misc
 from freqtrade.configuration import TimeRange
 from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv
-from freqtrade.exchange import Exchange, timeframe_to_minutes
+from freqtrade.exchange import Exchange, timeframe_to_minutes, timeframe_to_seconds
 
 logger = logging.getLogger(__name__)
 
@@ -127,7 +128,8 @@ def load_pair_history(pair: str,
                       refresh_pairs: bool = False,
                       exchange: Optional[Exchange] = None,
                       fill_up_missing: bool = True,
-                      drop_incomplete: bool = True
+                      drop_incomplete: bool = True,
+                      startup_candles: int = 0,
                       ) -> DataFrame:
     """
     Loads cached ticker history for the given pair.
@@ -140,9 +142,15 @@ def load_pair_history(pair: str,
     :param exchange: Exchange object (needed when using "refresh_pairs")
     :param fill_up_missing: Fill missing values with "No action"-candles
     :param drop_incomplete: Drop last candle assuming it may be incomplete.
+    :param startup_candles: Additional candles to load at the start of the period
     :return: DataFrame with ohlcv data
     """
 
+    timerange_startup = deepcopy(timerange)
+    if startup_candles:
+        logger.info('Using indicator startup period: %s ...', startup_candles)
+        timerange_startup.subtract_start(timeframe_to_seconds(ticker_interval) * startup_candles)
+
     # The user forced the refresh of pairs
     if refresh_pairs:
         download_pair_history(datadir=datadir,
@@ -151,11 +159,11 @@ def load_pair_history(pair: str,
                               ticker_interval=ticker_interval,
                               timerange=timerange)
 
-    pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
+    pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange_startup)
 
     if pairdata:
-        if timerange:
+        if timerange_startup:
-            _validate_pairdata(pair, pairdata, timerange)
+            _validate_pairdata(pair, pairdata, timerange_startup)
         return parse_ticker_dataframe(pairdata, ticker_interval, pair=pair,
                                       fill_missing=fill_up_missing,
                                       drop_incomplete=drop_incomplete)
@@ -174,10 +182,20 @@ def load_data(datadir: Path,
               exchange: Optional[Exchange] = None,
               timerange: Optional[TimeRange] = None,
               fill_up_missing: bool = True,
+              startup_candles: int = 0,
               ) -> Dict[str, DataFrame]:
     """
     Loads ticker history data for a list of pairs
-    :return: dict(<pair>:<tickerlist>)
+    :param datadir: Path to the data storage location.
+    :param ticker_interval: Ticker-interval (e.g. "5m")
+    :param pairs: List of pairs to load
+    :param refresh_pairs: Refresh pairs from exchange.
+        (Note: Requires exchange to be passed as well.)
+    :param exchange: Exchange object (needed when using "refresh_pairs")
+    :param timerange: Limit data to be loaded to this timerange
+    :param fill_up_missing: Fill missing values with "No action"-candles
+    :param startup_candles: Additional candles to load at the start of the period
+    :return: dict(<pair>:<Dataframe>)
     TODO: refresh_pairs is still used by edge to keep the data uptodate.
         This should be replaced in the future. Instead, writing the current candles to disk
         from dataprovider should be implemented, as this would avoid loading ohlcv data twice.

freqtrade/optimize/backtesting.py

@@ -92,7 +92,6 @@ class Backtesting:
 
         # Get maximum required startup period
         self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
-        self.required_startup_s = self.required_startup * timeframe_to_seconds(self.ticker_interval)
         # Load one (first) strategy
         self._set_strategy(self.strategylist[0])
 
@@ -422,11 +421,6 @@ class Backtesting:
         timerange = TimeRange.parse_timerange(None if self.config.get(
             'timerange') is None else str(self.config.get('timerange')))
 
-        logger.info('Using indicator startup period: %s ...', self.required_startup)
-
-        # Timerange_startup is timerange - startup-candles
-        timerange_startup = deepcopy(timerange)
-        timerange_startup.subtract_start(self.required_startup_s)
 
         data = history.load_data(
             datadir=Path(self.config['datadir']),
@@ -453,10 +447,12 @@ class Backtesting:
             'Loading backtest data from %s up to %s (%s days)..',
             min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
         )
-        if not timerange_startup.starttype:
+        if not timerange.starttype:
             # If no startts was defined, we need to move the backtesting start
             logger.info("Moving start-date by %s candles.", self.required_startup)
-            timerange.startts = min_date.timestamp + self.required_startup_s
+            timerange.startts = (min_date.timestamp
+                                 + timeframe_to_seconds(self.ticker_interval)
+                                 * self.required_startup)
             timerange.starttype = 'date'
 
         for strat in self.strategylist:

tests/optimize/test_backtesting.py

@@ -117,7 +117,7 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None:
 
 
 def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False,
-                     timerange=None, exchange=None, live=False):
+                     timerange=None, exchange=None, live=False, startup_candles=0):
     tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange)
     pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', pair="UNITTEST/BTC",
                                                        fill_missing=True)}