From 141c454187c96bd8d2f0f85b4309a18e2a8b1bfb Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 20 Oct 2019 11:17:01 +0200
Subject: [PATCH 01/18] Add startup-candles-argument for strategy

---
 freqtrade/strategy/interface.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py
index 014ca9968..48a70b0ce 100644
--- a/freqtrade/strategy/interface.py
+++ b/freqtrade/strategy/interface.py
@@ -103,6 +103,9 @@ class IStrategy(ABC):
     # run "populate_indicators" only for new candle
     process_only_new_candles: bool = False
 
+    # Count of candles the strategy requires before producing valid signals
+    startup_candle_count: int = 0
+
     # Class level variables (intentional) containing
     # the dataprovider (dp) (access to other candles, historic data, ...)
     # and wallets - access to the current balance.
@@ -421,6 +424,7 @@ class IStrategy(ABC):
     def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]:
         """
         Creates a dataframe and populates indicators for given ticker data
+        Used by optimize operations only, not  during dry / live runs.
         """
         return {pair: self.advise_indicators(pair_data, {'pair': pair})
                 for pair, pair_data in tickerdata.items()}

From 616fe08bcea0dae438906aaff60434749ff4e375 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 20 Oct 2019 11:29:25 +0200
Subject: [PATCH 02/18] Add subtract_start to timerange object

---
 freqtrade/configuration/timerange.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py
index fc759ab6e..527402af7 100644
--- a/freqtrade/configuration/timerange.py
+++ b/freqtrade/configuration/timerange.py
@@ -27,6 +27,10 @@ class TimeRange:
         return (self.starttype == other.starttype and self.stoptype == other.stoptype
                 and self.startts == other.startts and self.stopts == other.stopts)
 
+    def subtract_start(self, seconds) -> None:
+        if self.startts:
+            self.startts = self.startts - seconds
+
     @staticmethod
     def parse_timerange(text: Optional[str]):
         """

From 9e7e051eb42d45fffed0b8318587a0f8e7e78d99 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 20 Oct 2019 11:30:20 +0200
Subject: [PATCH 03/18] add trim-dataframe method

---
 freqtrade/data/history.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py
index ed5d80b0e..d385a28ed 100644
--- a/freqtrade/data/history.py
+++ b/freqtrade/data/history.py
@@ -13,6 +13,7 @@ from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
 import arrow
+import pytz
 from pandas import DataFrame
 
 from freqtrade import OperationalException, misc
@@ -49,6 +50,19 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]:
     return tickerlist[start_index:stop_index]
 
 
+def trim_dataframe(df: DataFrame, timerange: TimeRange) -> DataFrame:
+    """
+    Trim dataframe based on given timerange
+    """
+    if timerange.starttype == 'date':
+        start = datetime.fromtimestamp(timerange.startts, tz=pytz.utc)
+        df = df.loc[df['date'] >= start, :]
+    if timerange.stoptype == 'date':
+        stop = datetime.fromtimestamp(timerange.stopts, tz=pytz.utc)
+        df = df.loc[df['date'] <= stop, :]
+    return df
+
+
 def load_tickerdata_file(datadir: Path, pair: str, ticker_interval: str,
                          timerange: Optional[TimeRange] = None) -> Optional[list]:
     """

From 9c7696a8ce2e4833c4995b7e2ff30368b6292913 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 20 Oct 2019 13:56:01 +0200
Subject: [PATCH 04/18] Add required_startup to backtesting

---
 freqtrade/optimize/backtesting.py | 33 ++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index fb8c182ee..aa8a6a882 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -15,7 +15,7 @@ from freqtrade import OperationalException
 from freqtrade.configuration import TimeRange
 from freqtrade.data import history
 from freqtrade.data.dataprovider import DataProvider
-from freqtrade.exchange import timeframe_to_minutes
+from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
 from freqtrade.misc import file_dump_json
 from freqtrade.persistence import Trade
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -90,6 +90,9 @@ class Backtesting:
         self.ticker_interval = str(self.config.get('ticker_interval'))
         self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval)
 
+        # Get maximum required startup period
+        self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
+        self.required_startup_s = self.required_startup * timeframe_to_seconds(self.ticker_interval)
         # Load one (first) strategy
         self._set_strategy(self.strategylist[0])
 
@@ -418,11 +421,19 @@ class Backtesting:
 
         timerange = TimeRange.parse_timerange(None if self.config.get(
             'timerange') is None else str(self.config.get('timerange')))
+
+        logger.info('Using indicator startup period: %s ...', self.required_startup)
+
+        # Timerange_startup is timerange - startup-candles
+        timerange_startup = deepcopy(timerange)
+        timerange_startup.subtract_start(self.required_startup_s)
+
         data = history.load_data(
             datadir=Path(self.config['datadir']),
             pairs=pairs,
             ticker_interval=self.ticker_interval,
             timerange=timerange,
+            startup_candles=self.required_startup
         )
 
         if not data:
@@ -439,11 +450,14 @@ class Backtesting:
         min_date, max_date = history.get_timeframe(data)
 
         logger.info(
-            'Backtesting with data from %s up to %s (%s days)..',
-            min_date.isoformat(),
-            max_date.isoformat(),
-            (max_date - min_date).days
+            'Loading backtest data from %s up to %s (%s days)..',
+            min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
         )
+        if not timerange_startup.starttype:
+            # If no startts was defined, we need to move the backtesting start
+            logger.info("Moving start-date by %s candles.", self.required_startup)
+            timerange.startts = min_date.timestamp + self.required_startup_s
+            timerange.starttype = 'date'
 
         for strat in self.strategylist:
             logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
@@ -452,6 +466,15 @@ class Backtesting:
             # need to reprocess data every time to populate signals
             preprocessed = self.strategy.tickerdata_to_dataframe(data)
 
+            # Trim startup period from analyzed dataframe
+            for pair, df in preprocessed.items():
+                preprocessed[pair] = history.trim_dataframe(df, timerange)
+            min_date, max_date = history.get_timeframe(preprocessed)
+
+            logger.info(
+                'Backtesting with data from %s up to %s (%s days)..',
+                min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
+            )
             # Execute backtest and print results
             all_results[self.strategy.get_strategy_name()] = self.backtest(
                 {

From 704121c197798c2c0ad3829ff464a997d60fa1ff Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 20 Oct 2019 14:02:53 +0200
Subject: [PATCH 05/18] Move most logic to history

---
 freqtrade/data/history.py          | 30 ++++++++++++++++++++++++------
 freqtrade/optimize/backtesting.py  | 12 ++++--------
 tests/optimize/test_backtesting.py |  2 +-
 3 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py
index d385a28ed..71ac5c9a7 100644
--- a/freqtrade/data/history.py
+++ b/freqtrade/data/history.py
@@ -8,6 +8,7 @@ Includes:
 
 import logging
 import operator
+from copy import deepcopy
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
@@ -19,7 +20,7 @@ from pandas import DataFrame
 from freqtrade import OperationalException, misc
 from freqtrade.configuration import TimeRange
 from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv
-from freqtrade.exchange import Exchange, timeframe_to_minutes
+from freqtrade.exchange import Exchange, timeframe_to_minutes, timeframe_to_seconds
 
 logger = logging.getLogger(__name__)
 
@@ -127,7 +128,8 @@ def load_pair_history(pair: str,
                       refresh_pairs: bool = False,
                       exchange: Optional[Exchange] = None,
                       fill_up_missing: bool = True,
-                      drop_incomplete: bool = True
+                      drop_incomplete: bool = True,
+                      startup_candles: int = 0,
                       ) -> DataFrame:
     """
     Loads cached ticker history for the given pair.
@@ -140,9 +142,15 @@ def load_pair_history(pair: str,
     :param exchange: Exchange object (needed when using "refresh_pairs")
     :param fill_up_missing: Fill missing values with "No action"-candles
     :param drop_incomplete: Drop last candle assuming it may be incomplete.
+    :param startup_candles: Additional candles to load at the start of the period
     :return: DataFrame with ohlcv data
     """
 
+    timerange_startup = deepcopy(timerange)
+    if startup_candles:
+        logger.info('Using indicator startup period: %s ...', startup_candles)
+        timerange_startup.subtract_start(timeframe_to_seconds(ticker_interval) * startup_candles)
+
     # The user forced the refresh of pairs
     if refresh_pairs:
         download_pair_history(datadir=datadir,
@@ -151,11 +159,11 @@ def load_pair_history(pair: str,
                               ticker_interval=ticker_interval,
                               timerange=timerange)
 
-    pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
+    pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange_startup)
 
     if pairdata:
-        if timerange:
-            _validate_pairdata(pair, pairdata, timerange)
+        if timerange_startup:
+            _validate_pairdata(pair, pairdata, timerange_startup)
         return parse_ticker_dataframe(pairdata, ticker_interval, pair=pair,
                                       fill_missing=fill_up_missing,
                                       drop_incomplete=drop_incomplete)
@@ -174,10 +182,20 @@ def load_data(datadir: Path,
               exchange: Optional[Exchange] = None,
               timerange: Optional[TimeRange] = None,
               fill_up_missing: bool = True,
+              startup_candles: int = 0,
               ) -> Dict[str, DataFrame]:
     """
     Loads ticker history data for a list of pairs
-    :return: dict(<pair>:<tickerlist>)
+    :param datadir: Path to the data storage location.
+    :param ticker_interval: Ticker-interval (e.g. "5m")
+    :param pairs: List of pairs to load
+    :param refresh_pairs: Refresh pairs from exchange.
+        (Note: Requires exchange to be passed as well.)
+    :param exchange: Exchange object (needed when using "refresh_pairs")
+    :param timerange: Limit data to be loaded to this timerange
+    :param fill_up_missing: Fill missing values with "No action"-candles
+    :param startup_candles: Additional candles to load at the start of the period
+    :return: dict(<pair>:<Dataframe>)
     TODO: refresh_pairs is still used by edge to keep the data uptodate.
         This should be replaced in the future. Instead, writing the current candles to disk
         from dataprovider should be implemented, as this would avoid loading ohlcv data twice.
diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index aa8a6a882..59130dbc0 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -92,7 +92,6 @@ class Backtesting:
 
         # Get maximum required startup period
         self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
-        self.required_startup_s = self.required_startup * timeframe_to_seconds(self.ticker_interval)
         # Load one (first) strategy
         self._set_strategy(self.strategylist[0])
 
@@ -422,11 +421,6 @@ class Backtesting:
         timerange = TimeRange.parse_timerange(None if self.config.get(
             'timerange') is None else str(self.config.get('timerange')))
 
-        logger.info('Using indicator startup period: %s ...', self.required_startup)
-
-        # Timerange_startup is timerange - startup-candles
-        timerange_startup = deepcopy(timerange)
-        timerange_startup.subtract_start(self.required_startup_s)
 
         data = history.load_data(
             datadir=Path(self.config['datadir']),
@@ -453,10 +447,12 @@ class Backtesting:
             'Loading backtest data from %s up to %s (%s days)..',
             min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
         )
-        if not timerange_startup.starttype:
+        if not timerange.starttype:
             # If no startts was defined, we need to move the backtesting start
             logger.info("Moving start-date by %s candles.", self.required_startup)
-            timerange.startts = min_date.timestamp + self.required_startup_s
+            timerange.startts = (min_date.timestamp
+                                 + timeframe_to_seconds(self.ticker_interval)
+                                 * self.required_startup)
             timerange.starttype = 'date'
 
         for strat in self.strategylist:
diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py
index 998edda8a..3353274ef 100644
--- a/tests/optimize/test_backtesting.py
+++ b/tests/optimize/test_backtesting.py
@@ -117,7 +117,7 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None:
 
 
 def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False,
-                     timerange=None, exchange=None, live=False):
+                     timerange=None, exchange=None, live=False, startup_candles=0):
     tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange)
     pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', pair="UNITTEST/BTC",
                                                        fill_missing=True)}

From 6382a4cd042ed1e84af48493c60e7229986a0e11 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 23 Oct 2019 17:57:38 +0200
Subject: [PATCH 06/18] Implement startup-period to default-strategy

---
 freqtrade/optimize/backtesting.py      |  9 +++++----
 freqtrade/strategy/default_strategy.py |  6 +++---
 tests/optimize/test_backtesting.py     | 10 ++++++++--
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 59130dbc0..6b82dd601 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -421,7 +421,6 @@ class Backtesting:
         timerange = TimeRange.parse_timerange(None if self.config.get(
             'timerange') is None else str(self.config.get('timerange')))
 
-
         data = history.load_data(
             datadir=Path(self.config['datadir']),
             pairs=pairs,
@@ -447,9 +446,11 @@ class Backtesting:
             'Loading backtest data from %s up to %s (%s days)..',
             min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
         )
-        if not timerange.starttype:
-            # If no startts was defined, we need to move the backtesting start
-            logger.info("Moving start-date by %s candles.", self.required_startup)
+        if (not timerange.starttype or (self.required_startup
+                                        and min_date.timestamp == timerange.startts)):
+            # If no startts was defined, or test-data starts at the defined test-date
+            logger.warning("Moving start-date by %s candles to account for startup time.",
+                           self.required_startup)
             timerange.startts = (min_date.timestamp
                                  + timeframe_to_seconds(self.ticker_interval)
                                  * self.required_startup)
diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py
index b839a9618..0a241691c 100644
--- a/freqtrade/strategy/default_strategy.py
+++ b/freqtrade/strategy/default_strategy.py
@@ -39,6 +39,9 @@ class DefaultStrategy(IStrategy):
         'stoploss_on_exchange': False
     }
 
+    # Count of candles the strategy requires before producing valid signals
+    startup_candle_count: int = 20
+
     # Optional time in force for orders
     order_time_in_force = {
         'buy': 'gtc',
@@ -105,9 +108,6 @@ class DefaultStrategy(IStrategy):
         # EMA - Exponential Moving Average
         dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10)
 
-        # SMA - Simple Moving Average
-        dataframe['sma'] = ta.SMA(dataframe, timeperiod=40)
-
         return dataframe
 
     def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py
index 3353274ef..b14209e2d 100644
--- a/tests/optimize/test_backtesting.py
+++ b/tests/optimize/test_backtesting.py
@@ -838,7 +838,10 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
         f'Using data directory: {testdatadir} ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
-        'Backtesting with data from 2017-11-14T21:17:00+00:00 '
+        'Backtesting with data from 2017-11-14T21:37:00+00:00 '
+        'up to 2017-11-14T22:58:00+00:00 (0 days)..',
+        'Moving start-date by 20 candles to account for startup time.',
+        'Loading backtest data from 2017-11-14T21:17:00+00:00 '
         'up to 2017-11-14T22:58:00+00:00 (0 days)..',
         'Parameter --enable-position-stacking detected ...'
     ]
@@ -892,7 +895,10 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
         f'Using data directory: {testdatadir} ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
-        'Backtesting with data from 2017-11-14T21:17:00+00:00 '
+        'Backtesting with data from 2017-11-14T21:37:00+00:00 '
+        'up to 2017-11-14T22:58:00+00:00 (0 days)..',
+        'Moving start-date by 20 candles to account for startup time.',
+        'Loading backtest data from 2017-11-14T21:17:00+00:00 '
         'up to 2017-11-14T22:58:00+00:00 (0 days)..',
         'Parameter --enable-position-stacking detected ...',
         'Running backtesting for Strategy DefaultStrategy',

From 5c2682e2c94828f1cbfa32b9ec6534fae103caf1 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 23 Oct 2019 17:58:26 +0200
Subject: [PATCH 07/18] Add startup_candle_count to sample strategy

---
 freqtrade/strategy/default_strategy.py  | 2 +-
 user_data/strategies/sample_strategy.py | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py
index 0a241691c..6c343b477 100644
--- a/freqtrade/strategy/default_strategy.py
+++ b/freqtrade/strategy/default_strategy.py
@@ -39,7 +39,7 @@ class DefaultStrategy(IStrategy):
         'stoploss_on_exchange': False
     }
 
-    # Count of candles the strategy requires before producing valid signals
+    # Number of candles the strategy requires before producing valid signals
     startup_candle_count: int = 20
 
     # Optional time in force for orders
diff --git a/user_data/strategies/sample_strategy.py b/user_data/strategies/sample_strategy.py
index 80c30283d..c2fd681d2 100644
--- a/user_data/strategies/sample_strategy.py
+++ b/user_data/strategies/sample_strategy.py
@@ -59,6 +59,9 @@ class SampleStrategy(IStrategy):
     sell_profit_only = False
     ignore_roi_if_buy_signal = False
 
+    # Number of candles the strategy requires before producing valid signals
+    startup_candle_count: int = 20
+
     # Optional order type mapping.
     order_types = {
         'buy': 'limit',

From bd4a23beeb9506b75b4556176ae3eef26924e0ab Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 23 Oct 2019 19:30:01 +0200
Subject: [PATCH 08/18] Refactor start-adjust logic to timerange

---
 freqtrade/configuration/timerange.py | 27 +++++++++++++++++++++++++++
 freqtrade/optimize/backtesting.py    | 12 +++---------
 2 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py
index 527402af7..5731631c5 100644
--- a/freqtrade/configuration/timerange.py
+++ b/freqtrade/configuration/timerange.py
@@ -1,11 +1,14 @@
 """
 This module contains the argument manager class
 """
+import logging
 import re
 from typing import Optional
 
 import arrow
 
+logger = logging.getLogger(__name__)
+
 
 class TimeRange:
     """
@@ -28,9 +31,33 @@ class TimeRange:
                 and self.startts == other.startts and self.stopts == other.stopts)
 
     def subtract_start(self, seconds) -> None:
+        """
+        Subtracts <seconds> from startts if startts is set.
+        :param seconds: Seconds to subtract from starttime
+        :return: None (Modifies the object in place)
+        """
         if self.startts:
             self.startts = self.startts - seconds
 
+    def adjust_start_if_necessary(self, ticker_interval_secs: int, startup_candles: int,
+                                  min_date: arrow.Arrow) -> None:
+        """
+        Adjust startts by <startup_candles> candles.
+        Applies only if no startup-candles have been available.
+        :param ticker_interval_secs: Ticker interval in seconds e.g. `timeframe_to_seconds('5m')`
+        :param startup_candles: Number of candles to move start-date forward
+        :param min_date: Minimum data date loaded. Key kriterium to decide if start-time
+                         has to be moved
+        :return: None (Modifies the object in place)
+        """
+        if (not self.starttype or (startup_candles
+                                   and min_date.timestamp == self.startts)):
+            # If no startts was defined, or test-data starts at the defined test-date
+            logger.warning("Moving start-date by %s candles to account for startup time.",
+                           startup_candles)
+            self.startts = (min_date.timestamp + ticker_interval_secs * startup_candles)
+            self.starttype = 'date'
+
     @staticmethod
     def parse_timerange(text: Optional[str]):
         """
diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 6b82dd601..1d6b328a8 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -446,15 +446,9 @@ class Backtesting:
             'Loading backtest data from %s up to %s (%s days)..',
             min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
         )
-        if (not timerange.starttype or (self.required_startup
-                                        and min_date.timestamp == timerange.startts)):
-            # If no startts was defined, or test-data starts at the defined test-date
-            logger.warning("Moving start-date by %s candles to account for startup time.",
-                           self.required_startup)
-            timerange.startts = (min_date.timestamp
-                                 + timeframe_to_seconds(self.ticker_interval)
-                                 * self.required_startup)
-            timerange.starttype = 'date'
+        # Adjust startts forward if not enough data is available
+        timerange.adjust_start_if_necessary(timeframe_to_seconds(self.ticker_interval),
+                                            self.required_startup, min_date)
 
         for strat in self.strategylist:
             logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())

From 5cdae17d19b75b07aed188fd676b056a0ec4e7a2 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 23 Oct 2019 19:41:10 +0200
Subject: [PATCH 09/18] Add tests for timerange modifications

---
 tests/test_timerange.py | 37 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/tests/test_timerange.py b/tests/test_timerange.py
index 4851cbebd..d758092ed 100644
--- a/tests/test_timerange.py
+++ b/tests/test_timerange.py
@@ -1,10 +1,11 @@
 # pragma pylint: disable=missing-docstring, C0103
+import arrow
 import pytest
 
 from freqtrade.configuration import TimeRange
 
 
-def test_parse_timerange_incorrect() -> None:
+def test_parse_timerange_incorrect():
 
     assert TimeRange('date', None, 1274486400, 0) == TimeRange.parse_timerange('20100522-')
     assert TimeRange(None, 'date', 0, 1274486400) == TimeRange.parse_timerange('-20100522')
@@ -28,3 +29,37 @@ def test_parse_timerange_incorrect() -> None:
 
     with pytest.raises(Exception, match=r'Incorrect syntax.*'):
         TimeRange.parse_timerange('-')
+
+
+def test_subtract_start():
+    x = TimeRange('date', 'date', 1274486400, 1438214400)
+    x.subtract_start(300)
+    assert x.startts == 1274486400 - 300
+
+    # Do nothing if no startdate exists
+    x = TimeRange(None, 'date', 0, 1438214400)
+    x.subtract_start(300)
+    assert not x.startts
+
+    x = TimeRange('date', None, 1274486400, 0)
+    x.subtract_start(300)
+    assert x.startts == 1274486400 - 300
+
+
+def test_adjust_start_if_necessary():
+    min_date = arrow.Arrow(2017, 11, 14, 21, 15, 00)
+
+    x = TimeRange('date', 'date', 1510694100, 1510780500)
+    # Adjust by 20 candles - min_date == startts
+    x.adjust_start_if_necessary(300, 20, min_date)
+    assert x.startts == 1510694100 + (20 * 300)
+
+    x = TimeRange('date', 'date', 1510700100, 1510780500)
+    # Do nothing, startupe is set and different min_date
+    x.adjust_start_if_necessary(300, 20, min_date)
+    assert x.startts == 1510694100 + (20 * 300)
+
+    x = TimeRange(None, 'date', 0, 1510780500)
+    # Adjust by 20 candles = 20 * 5m
+    x.adjust_start_if_necessary(300, 20, min_date)
+    assert x.startts == 1510694100 + (20 * 300)

From 86624411c6e5af7c6fa1c1dc9f184f5420ef1172 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 23 Oct 2019 19:54:43 +0200
Subject: [PATCH 10/18] Test trim_dataframe

---
 tests/data/test_history.py | 40 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/tests/data/test_history.py b/tests/data/test_history.py
index 95382768a..d7e0562cc 100644
--- a/tests/data/test_history.py
+++ b/tests/data/test_history.py
@@ -427,6 +427,46 @@ def test_trim_tickerlist(testdatadir) -> None:
     assert not ticker
 
 
+def test_trim_dataframe(testdatadir) -> None:
+    data = history.load_data(
+        datadir=testdatadir,
+        ticker_interval='1m',
+        pairs=['UNITTEST/BTC']
+    )['UNITTEST/BTC']
+    min_date = int(data.iloc[0]['date'].timestamp())
+    max_date = int(data.iloc[-1]['date'].timestamp())
+    data_modify = data.copy()
+
+    # Remove first 30 minutes (1800 s)
+    tr = TimeRange('date', None, min_date + 1800, 0)
+    data_modify = history.trim_dataframe(data_modify, tr)
+    assert not data_modify.equals(data)
+    assert len(data_modify) < len(data)
+    assert len(data_modify) == len(data) - 30
+    assert all(data_modify.iloc[-1] == data.iloc[-1])
+    assert all(data_modify.iloc[0] == data.iloc[30])
+
+    data_modify = data.copy()
+    # Remove last 30 minutes (1800 s)
+    tr = TimeRange(None, 'date', 0, max_date - 1800)
+    data_modify = history.trim_dataframe(data_modify, tr)
+    assert not data_modify.equals(data)
+    assert len(data_modify) < len(data)
+    assert len(data_modify) == len(data) - 30
+    assert all(data_modify.iloc[0] == data.iloc[0])
+    assert all(data_modify.iloc[-1] == data.iloc[-31])
+
+    data_modify = data.copy()
+    # Remove first 25 and last 30 minutes (1800 s)
+    tr = TimeRange('date', 'date', min_date + 1500, max_date - 1800)
+    data_modify = history.trim_dataframe(data_modify, tr)
+    assert not data_modify.equals(data)
+    assert len(data_modify) < len(data)
+    assert len(data_modify) == len(data) - 55
+    # first row matches 25th original row
+    assert all(data_modify.iloc[0] == data.iloc[25])
+
+
 def test_file_dump_json_tofile(testdatadir) -> None:
     file = testdatadir / 'test_{id}.json'.format(id=str(uuid.uuid4()))
     data = {'bar': 'foo'}

From 33164ac78ee2d72647931467abcb8d60e2d5e54e Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 23 Oct 2019 20:13:43 +0200
Subject: [PATCH 11/18] Refactor loading of bt data to backtesting ...

---
 freqtrade/data/history.py         |  8 ++++-
 freqtrade/optimize/backtesting.py | 55 ++++++++++++++++---------------
 freqtrade/optimize/hyperopt.py    | 25 ++++----------
 3 files changed, 42 insertions(+), 46 deletions(-)

diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py
index 71ac5c9a7..dfd175b1f 100644
--- a/freqtrade/data/history.py
+++ b/freqtrade/data/history.py
@@ -183,6 +183,7 @@ def load_data(datadir: Path,
               timerange: Optional[TimeRange] = None,
               fill_up_missing: bool = True,
               startup_candles: int = 0,
+              fail_without_data: bool = False
               ) -> Dict[str, DataFrame]:
     """
     Loads ticker history data for a list of pairs
@@ -195,6 +196,7 @@ def load_data(datadir: Path,
     :param timerange: Limit data to be loaded to this timerange
     :param fill_up_missing: Fill missing values with "No action"-candles
     :param startup_candles: Additional candles to load at the start of the period
+    :param fail_without_data: Raise OperationalException if no data is found.
     :return: dict(<pair>:<Dataframe>)
     TODO: refresh_pairs is still used by edge to keep the data uptodate.
         This should be replaced in the future. Instead, writing the current candles to disk
@@ -208,9 +210,13 @@ def load_data(datadir: Path,
                                  datadir=datadir, timerange=timerange,
                                  refresh_pairs=refresh_pairs,
                                  exchange=exchange,
-                                 fill_up_missing=fill_up_missing)
+                                 fill_up_missing=fill_up_missing,
+                                 startup_candles=startup_candles)
         if hist is not None:
             result[pair] = hist
+
+    if fail_without_data and not result:
+        raise OperationalException("No data found. Terminating.")
     return result
 
 
diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 1d6b328a8..fe31912bc 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -105,6 +105,31 @@ class Backtesting:
         # And the regular "stoploss" function would not apply to that case
         self.strategy.order_types['stoploss_on_exchange'] = False
 
+    def load_bt_data(self):
+        timerange = TimeRange.parse_timerange(None if self.config.get(
+            'timerange') is None else str(self.config.get('timerange')))
+
+        data = history.load_data(
+            datadir=Path(self.config['datadir']),
+            pairs=self.config['exchange']['pair_whitelist'],
+            ticker_interval=self.ticker_interval,
+            timerange=timerange,
+            startup_candles=self.required_startup,
+            fail_without_data=True,
+        )
+
+        min_date, max_date = history.get_timeframe(data)
+
+        logger.info(
+            'Loading data from %s up to %s (%s days)..',
+            min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
+        )
+        # Adjust startts forward if not enough data is available
+        timerange.adjust_start_if_necessary(timeframe_to_seconds(self.ticker_interval),
+                                            self.required_startup, min_date)
+
+        return data, timerange
+
     def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame,
                              skip_nan: bool = False) -> str:
         """
@@ -414,42 +439,18 @@ class Backtesting:
         :return: None
         """
         data: Dict[str, Any] = {}
-        pairs = self.config['exchange']['pair_whitelist']
         logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
         logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
-
-        timerange = TimeRange.parse_timerange(None if self.config.get(
-            'timerange') is None else str(self.config.get('timerange')))
-
-        data = history.load_data(
-            datadir=Path(self.config['datadir']),
-            pairs=pairs,
-            ticker_interval=self.ticker_interval,
-            timerange=timerange,
-            startup_candles=self.required_startup
-        )
-
-        if not data:
-            logger.critical("No data found. Terminating.")
-            return
         # Use max_open_trades in backtesting, except --disable-max-market-positions is set
         if self.config.get('use_max_market_positions', True):
             max_open_trades = self.config['max_open_trades']
         else:
             logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...')
             max_open_trades = 0
+
+        data, timerange = self.load_bt_data()
+
         all_results = {}
-
-        min_date, max_date = history.get_timeframe(data)
-
-        logger.info(
-            'Loading backtest data from %s up to %s (%s days)..',
-            min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
-        )
-        # Adjust startts forward if not enough data is available
-        timerange.adjust_start_if_necessary(timeframe_to_seconds(self.ticker_interval),
-                                            self.required_startup, min_date)
-
         for strat in self.strategylist:
             logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
             self._set_strategy(strat)
diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 07258a048..2264234d4 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -23,7 +23,7 @@ from skopt import Optimizer
 from skopt.space import Dimension
 
 from freqtrade.configuration import TimeRange
-from freqtrade.data.history import load_data, get_timeframe
+from freqtrade.data.history import load_data, get_timeframe, trim_dataframe
 from freqtrade.misc import round_dict
 from freqtrade.optimize.backtesting import Backtesting
 # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
@@ -379,30 +379,19 @@ class Hyperopt:
             )
 
     def start(self) -> None:
-        timerange = TimeRange.parse_timerange(None if self.config.get(
-            'timerange') is None else str(self.config.get('timerange')))
-        data = load_data(
-            datadir=Path(self.config['datadir']),
-            pairs=self.config['exchange']['pair_whitelist'],
-            ticker_interval=self.backtesting.ticker_interval,
-            timerange=timerange
-        )
+        data, timerange = self.backtesting.load_bt_data()
 
-        if not data:
-            logger.critical("No data found. Terminating.")
-            return
+        preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data)
 
+        # Trim startup period from analyzed dataframe
+        for pair, df in preprocessed.items():
+            preprocessed[pair] = trim_dataframe(df, timerange)
         min_date, max_date = get_timeframe(data)
 
         logger.info(
             'Hyperopting with data from %s up to %s (%s days)..',
-            min_date.isoformat(),
-            max_date.isoformat(),
-            (max_date - min_date).days
+            min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
         )
-
-        preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data)
-
         dump(preprocessed, self.tickerdata_pickle)
 
         # We don't need exchange instance anymore while running hyperopt

From 2ba388074e577278a83a17eb92e3980176f75b56 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 23 Oct 2019 20:27:51 +0200
Subject: [PATCH 12/18] Fix small bugs

---
 freqtrade/configuration/timerange.py |  2 +-
 tests/optimize/test_backtesting.py   | 20 ++++++++------------
 tests/optimize/test_hyperopt.py      |  7 +++----
 3 files changed, 12 insertions(+), 17 deletions(-)

diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py
index 5731631c5..df5c937cf 100644
--- a/freqtrade/configuration/timerange.py
+++ b/freqtrade/configuration/timerange.py
@@ -51,7 +51,7 @@ class TimeRange:
         :return: None (Modifies the object in place)
         """
         if (not self.starttype or (startup_candles
-                                   and min_date.timestamp == self.startts)):
+                                   and min_date.timestamp >= self.startts)):
             # If no startts was defined, or test-data starts at the defined test-date
             logger.warning("Moving start-date by %s candles to account for startup time.",
                            startup_candles)
diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py
index b14209e2d..ba87848ec 100644
--- a/tests/optimize/test_backtesting.py
+++ b/tests/optimize/test_backtesting.py
@@ -117,7 +117,7 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None:
 
 
 def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False,
-                     timerange=None, exchange=None, live=False, startup_candles=0):
+                     timerange=None, exchange=None, live=False, *args, **kwargs):
     tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange)
     pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', pair="UNITTEST/BTC",
                                                        fill_missing=True)}
@@ -494,7 +494,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
     def get_timeframe(input1):
         return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
 
-    mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={}))
+    mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=None))
     mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe)
     mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
     patch_exchange(mocker)
@@ -511,10 +511,8 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
     default_conf['timerange'] = '20180101-20180102'
 
     backtesting = Backtesting(default_conf)
-    backtesting.start()
-    # check the logs, that will contain the backtest result
-
-    assert log_has('No data found. Terminating.', caplog)
+    with pytest.raises(OperationalException, match='No data found. Terminating.'):
+        backtesting.start()
 
 
 def test_backtest(default_conf, fee, mocker, testdatadir) -> None:
@@ -838,10 +836,9 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
         f'Using data directory: {testdatadir} ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
-        'Backtesting with data from 2017-11-14T21:37:00+00:00 '
+        'Loading data from 2017-11-14T20:57:00+00:00 '
         'up to 2017-11-14T22:58:00+00:00 (0 days)..',
-        'Moving start-date by 20 candles to account for startup time.',
-        'Loading backtest data from 2017-11-14T21:17:00+00:00 '
+        'Backtesting with data from 2017-11-14T21:17:00+00:00 '
         'up to 2017-11-14T22:58:00+00:00 (0 days)..',
         'Parameter --enable-position-stacking detected ...'
     ]
@@ -895,10 +892,9 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
         f'Using data directory: {testdatadir} ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
-        'Backtesting with data from 2017-11-14T21:37:00+00:00 '
+        'Loading data from 2017-11-14T20:57:00+00:00 '
         'up to 2017-11-14T22:58:00+00:00 (0 days)..',
-        'Moving start-date by 20 candles to account for startup time.',
-        'Loading backtest data from 2017-11-14T21:17:00+00:00 '
+        'Backtesting with data from 2017-11-14T21:17:00+00:00 '
         'up to 2017-11-14T22:58:00+00:00 (0 days)..',
         'Parameter --enable-position-stacking detected ...',
         'Running backtesting for Strategy DefaultStrategy',
diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py
index 052c3ba77..d1448d367 100644
--- a/tests/optimize/test_hyperopt.py
+++ b/tests/optimize/test_hyperopt.py
@@ -228,7 +228,7 @@ def test_start(mocker, default_conf, caplog) -> None:
 
 def test_start_no_data(mocker, default_conf, caplog) -> None:
     patched_configuration_load_config_file(mocker, default_conf)
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock(return_value={}))
+    mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=None))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -242,9 +242,8 @@ def test_start_no_data(mocker, default_conf, caplog) -> None:
         '--epochs', '5'
     ]
     args = get_args(args)
-    start_hyperopt(args)
-
-    assert log_has('No data found. Terminating.', caplog)
+    with pytest.raises(OperationalException, match='No data found. Terminating.'):
+        start_hyperopt(args)
 
 
 def test_start_filelock(mocker, default_conf, caplog) -> None:

From 2bc74882e9940dd6e5c67e94240e7031ed2a9ec1 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 27 Oct 2019 10:00:44 +0100
Subject: [PATCH 13/18] Add test for startup_candles

---
 tests/data/test_history.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/data/test_history.py b/tests/data/test_history.py
index d7e0562cc..057524fb3 100644
--- a/tests/data/test_history.py
+++ b/tests/data/test_history.py
@@ -95,6 +95,23 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> N
     _clean_test_file(file)
 
 
+def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None:
+    ltfmock = mocker.patch('freqtrade.data.history.load_tickerdata_file',
+                           MagicMock(return_value=None))
+    timerange = TimeRange('date', None, 1510639620, 0)
+    history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='1m',
+                              datadir=testdatadir, timerange=timerange,
+                              startup_candles=20,
+                              )
+    assert log_has(
+        'Using indicator startup period: 20 ...', caplog
+    )
+    assert ltfmock.call_count == 1
+    assert ltfmock.call_args_list[0][1]['timerange'] != timerange
+    # startts is 20 minutes earlier
+    assert ltfmock.call_args_list[0][1]['timerange'].startts == timerange.startts - 20 * 60
+
+
 def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog,
                                       default_conf, testdatadir) -> None:
     """

From c4cb098d14cafe4760551974b6de6c309888166e Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 27 Oct 2019 10:17:02 +0100
Subject: [PATCH 14/18] Update documentation with indicator_startup_period

---
 docs/backtesting.md            |  2 ++
 docs/strategy-customization.md | 35 ++++++++++++++++++++++++++++++++--
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/docs/backtesting.md b/docs/backtesting.md
index 34c5f1fbe..6d21fa2bf 100644
--- a/docs/backtesting.md
+++ b/docs/backtesting.md
@@ -72,6 +72,8 @@ The exported trades can be used for [further analysis](#further-backtest-result-
 freqtrade backtesting --export trades --export-filename=backtest_samplestrategy.json
 ```
 
+Please also read about the [strategy startup period](strategy-customization.md#strategy-startup-period).
+
 #### Supplying custom fee value
 
 Sometimes your account has certain fee rebates (fee reductions starting with a certain account size or monthly volume), which are not visible to ccxt.
diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md
index ab7dcfc30..ae44d32ea 100644
--- a/docs/strategy-customization.md
+++ b/docs/strategy-customization.md
@@ -117,6 +117,37 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame
     Look into the [user_data/strategies/sample_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/sample_strategy.py).
     Then uncomment indicators you need.
 
+### Strategy startup period
+
+Most indicators have an "instable period", in which they are either not available, or the calculation is incorrect. This can lead to inconsistencies, since Freqtrade does not know how long this instable period should be.
+To account for this, the strategy has an attribute, `startup_candle_count`. 
+This should be set to the maximum number of candles that the strategy requires to calculate stable indicators.
+
+In this example strategy, this should be set to 100 (`startup_candle_count = 100`), since the longest needed history is 100 candles.
+
+``` python
+    dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100)
+```
+
+By letting the bot know how much history is needed, backtest trades can start at the specified timerange during backtesting and hyperopt.
+
+!!! Warning:
+    `startup_candle_count` should be below `ohlcv_candle_limit` (which is 500 for most exchanges) - since only this amount of candles will be available during trading operations.
+
+#### Example
+
+Let's try to backtest 1 month (January 2019) of 5m candles.
+
+``` bash
+freqtrade backtesting --timerange 20190101-20190201 --ticker-interval 5m
+```
+
+Since backtesting knows it needs 100 candles to generate valid buy-signals, it'll load data from `20190101 - (100 * 5m)` - which is ~2019-12-31 15:30:00.
+If this data is available, Indicators will be calculated with this extended timerange. The startup period (Up to 2019-01-01 00:00:00) will then be removed before starting backtesting.
+
+!!! Note
+    If data for the startup-period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-01 08:30:00.
+
 ### Buy signal rules
 
 Edit the method `populate_buy_trend()` in your strategy file to update your buy strategy.
@@ -267,10 +298,10 @@ class Awesomestrategy(IStrategy):
 ```
 
 !!! Warning
-  The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
+    The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
 
 !!! Note
-  If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
+    If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
 
 ### Additional data (DataProvider)
 

From 223f0cd4d3780a348f1a743329cf671b38ec914d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 27 Oct 2019 10:18:58 +0100
Subject: [PATCH 15/18] Apply startup_period to edge as well

---
 freqtrade/edge/__init__.py | 3 ++-
 tests/edge/test_edge.py    | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py
index 2655fbc65..883bf4a0f 100644
--- a/freqtrade/edge/__init__.py
+++ b/freqtrade/edge/__init__.py
@@ -100,7 +100,8 @@ class Edge:
             ticker_interval=self.strategy.ticker_interval,
             refresh_pairs=self._refresh_pairs,
             exchange=self.exchange,
-            timerange=self._timerange
+            timerange=self._timerange,
+            startup_candles=self.strategy.startup_candle_count,
         )
 
         if not data:
diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py
index 5e244a97e..e1af50768 100644
--- a/tests/edge/test_edge.py
+++ b/tests/edge/test_edge.py
@@ -256,7 +256,7 @@ def test_edge_heartbeat_calculate(mocker, edge_conf):
 
 
 def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False,
-                     timerange=None, exchange=None):
+                     timerange=None, exchange=None, *args, **kwargs):
     hz = 0.1
     base = 0.001
 

From 73f5bff9c5c9487457f529a007cef99faff6f1f0 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 27 Oct 2019 10:38:21 +0100
Subject: [PATCH 16/18] Add validation to make sure strategies work on that
 exchange

---
 freqtrade/exchange/exchange.py           | 10 ++++++++++
 freqtrade/resolvers/strategy_resolver.py |  1 +
 tests/exchange/test_exchange.py          | 14 ++++++++++++++
 3 files changed, 25 insertions(+)

diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index 71f0737ef..3fd7d615c 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -228,6 +228,7 @@ class Exchange:
             self.validate_pairs(config['exchange']['pair_whitelist'])
             self.validate_ordertypes(config.get('order_types', {}))
             self.validate_order_time_in_force(config.get('order_time_in_force', {}))
+            self.validate_required_startup_candles(config.get('startup_candle_count', 0))
 
         # Converts the interval provided in minutes in config to seconds
         self.markets_refresh_interval: int = exchange_config.get(
@@ -443,6 +444,15 @@ class Exchange:
             raise OperationalException(
                 f'Time in force policies are not supported for {self.name} yet.')
 
+    def validate_required_startup_candles(self, startup_candles) -> None:
+        """
+        Checks if required startup_candles is more than ohlcv_candle_limit.
+        """
+        if startup_candles + 5 > self._ft_has['ohlcv_candle_limit']:
+            raise OperationalException(
+                f"This strategy requires {startup_candles} candles to start. "
+                f"{self.name} only provides {self._ft_has['ohlcv_candle_limit']}.")
+
     def exchange_has(self, endpoint: str) -> bool:
         """
         Checks if exchange implements a specific API endpoint.
diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py
index d6fbe9a7a..5bea74027 100644
--- a/freqtrade/resolvers/strategy_resolver.py
+++ b/freqtrade/resolvers/strategy_resolver.py
@@ -57,6 +57,7 @@ class StrategyResolver(IResolver):
                       ("order_time_in_force",             None,        False),
                       ("stake_currency",                  None,        False),
                       ("stake_amount",                    None,        False),
+                      ("startup_candle_count",            None,        False),
                       ("use_sell_signal",                 True,        True),
                       ("sell_profit_only",                False,       True),
                       ("ignore_roi_if_buy_signal",        False,       True),
diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py
index 1e0a5fdc3..e0e0cc7b1 100644
--- a/tests/exchange/test_exchange.py
+++ b/tests/exchange/test_exchange.py
@@ -533,6 +533,20 @@ def test_validate_order_types_not_in_config(default_conf, mocker):
     Exchange(conf)
 
 
+def test_validate_required_startup_candles(default_conf, mocker, caplog):
+    api_mock = MagicMock()
+    default_conf['startup_candle_count'] = 2000
+    mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance'))
+
+    mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
+    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock())
+
+    with pytest.raises(OperationalException, match=r'This strategy requires 2000.*'):
+        Exchange(default_conf)
+
+
 def test_exchange_has(default_conf, mocker):
     exchange = get_patched_exchange(mocker, default_conf)
     assert not exchange.exchange_has('ASDFASDF')

From 132a4da7cf6ce448877506d28e7fbc9b2b63701d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 27 Oct 2019 10:56:38 +0100
Subject: [PATCH 17/18] Small style fixes and adjusted tests

---
 docs/strategy-customization.md  |  2 +-
 freqtrade/data/history.py       |  2 +-
 freqtrade/exchange/exchange.py  |  1 +
 freqtrade/optimize/hyperopt.py  |  3 +--
 freqtrade/strategy/interface.py |  2 +-
 tests/exchange/test_exchange.py |  8 ++++++--
 tests/optimize/test_hyperopt.py | 24 ++++++++++++++++--------
 7 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md
index ae44d32ea..4d3d9bce5 100644
--- a/docs/strategy-customization.md
+++ b/docs/strategy-customization.md
@@ -131,7 +131,7 @@ In this example strategy, this should be set to 100 (`startup_candle_count = 100
 
 By letting the bot know how much history is needed, backtest trades can start at the specified timerange during backtesting and hyperopt.
 
-!!! Warning:
+!!! Warning
     `startup_candle_count` should be below `ohlcv_candle_limit` (which is 500 for most exchanges) - since only this amount of candles will be available during trading operations.
 
 #### Example
diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py
index dfd175b1f..c07b58da2 100644
--- a/freqtrade/data/history.py
+++ b/freqtrade/data/history.py
@@ -147,7 +147,7 @@ def load_pair_history(pair: str,
     """
 
     timerange_startup = deepcopy(timerange)
-    if startup_candles:
+    if startup_candles and timerange_startup:
         logger.info('Using indicator startup period: %s ...', startup_candles)
         timerange_startup.subtract_start(timeframe_to_seconds(ticker_interval) * startup_candles)
 
diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index 3fd7d615c..023e16cc5 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -447,6 +447,7 @@ class Exchange:
     def validate_required_startup_candles(self, startup_candles) -> None:
         """
         Checks if required startup_candles is more than ohlcv_candle_limit.
+        Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default.
         """
         if startup_candles + 5 > self._ft_has['ohlcv_candle_limit']:
             raise OperationalException(
diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 2264234d4..c576ea6f8 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -22,8 +22,7 @@ from pandas import DataFrame
 from skopt import Optimizer
 from skopt.space import Dimension
 
-from freqtrade.configuration import TimeRange
-from freqtrade.data.history import load_data, get_timeframe, trim_dataframe
+from freqtrade.data.history import get_timeframe, trim_dataframe
 from freqtrade.misc import round_dict
 from freqtrade.optimize.backtesting import Backtesting
 # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py
index 48a70b0ce..d42f8e989 100644
--- a/freqtrade/strategy/interface.py
+++ b/freqtrade/strategy/interface.py
@@ -424,7 +424,7 @@ class IStrategy(ABC):
     def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]:
         """
         Creates a dataframe and populates indicators for given ticker data
-        Used by optimize operations only, not  during dry / live runs.
+        Used by optimize operations only, not during dry / live runs.
         """
         return {pair: self.advise_indicators(pair_data, {'pair': pair})
                 for pair, pair_data in tickerdata.items()}
diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py
index e0e0cc7b1..6805d8e73 100644
--- a/tests/exchange/test_exchange.py
+++ b/tests/exchange/test_exchange.py
@@ -535,7 +535,6 @@ def test_validate_order_types_not_in_config(default_conf, mocker):
 
 def test_validate_required_startup_candles(default_conf, mocker, caplog):
     api_mock = MagicMock()
-    default_conf['startup_candle_count'] = 2000
     mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance'))
 
     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
@@ -543,7 +542,12 @@ def test_validate_required_startup_candles(default_conf, mocker, caplog):
     mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
     mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock())
 
-    with pytest.raises(OperationalException, match=r'This strategy requires 2000.*'):
+    default_conf['startup_candle_count'] = 20
+    ex = Exchange(default_conf)
+    assert ex
+    default_conf['startup_candle_count'] = 600
+
+    with pytest.raises(OperationalException, match=r'This strategy requires 600.*'):
         Exchange(default_conf)
 
 
diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py
index d1448d367..d0c37c40d 100644
--- a/tests/optimize/test_hyperopt.py
+++ b/tests/optimize/test_hyperopt.py
@@ -392,7 +392,8 @@ def test_roi_table_generation(hyperopt) -> None:
 
 def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -607,7 +608,8 @@ def test_continue_hyperopt(mocker, default_conf, caplog):
 
 def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -644,7 +646,8 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
 
 def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -681,7 +684,8 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) ->
 
 def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -727,7 +731,8 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys)
 
 def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) -> None:
     mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -756,7 +761,8 @@ def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) -
 
 def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -801,7 +807,8 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None:
 
 def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
@@ -852,7 +859,8 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None
 ])
 def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, method, space) -> None:
     mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
-    mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timeframe',
         MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))

From 61c037f2cf0bb6a43026bc43ad3e2ed4312d0d2b Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 28 Oct 2019 13:05:54 +0100
Subject: [PATCH 18/18] Fix some typos and comment mistakes

---
 docs/strategy-customization.md       | 14 +++++++-------
 freqtrade/configuration/timerange.py |  2 +-
 freqtrade/data/history.py            |  2 +-
 tests/test_timerange.py              |  2 +-
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md
index 4d3d9bce5..cef362ffd 100644
--- a/docs/strategy-customization.md
+++ b/docs/strategy-customization.md
@@ -119,8 +119,8 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame
 
 ### Strategy startup period
 
-Most indicators have an "instable period", in which they are either not available, or the calculation is incorrect. This can lead to inconsistencies, since Freqtrade does not know how long this instable period should be.
-To account for this, the strategy has an attribute, `startup_candle_count`. 
+Most indicators have an instable startup period, in which they are either not available, or the calculation is incorrect. This can lead to inconsistencies, since Freqtrade does not know how long this instable period should be.
+To account for this, the strategy can be assigned the `startup_candle_count` attribute.
 This should be set to the maximum number of candles that the strategy requires to calculate stable indicators.
 
 In this example strategy, this should be set to 100 (`startup_candle_count = 100`), since the longest needed history is 100 candles.
@@ -132,21 +132,21 @@ In this example strategy, this should be set to 100 (`startup_candle_count = 100
 By letting the bot know how much history is needed, backtest trades can start at the specified timerange during backtesting and hyperopt.
 
 !!! Warning
-    `startup_candle_count` should be below `ohlcv_candle_limit` (which is 500 for most exchanges) - since only this amount of candles will be available during trading operations.
+    `startup_candle_count` should be below `ohlcv_candle_limit` (which is 500 for most exchanges) - since only this amount of candles will be available during Dry-Run/Live Trade operations.
 
 #### Example
 
-Let's try to backtest 1 month (January 2019) of 5m candles.
+Let's try to backtest 1 month (January 2019) of 5m candles using the an example strategy with EMA100, as above.
 
 ``` bash
 freqtrade backtesting --timerange 20190101-20190201 --ticker-interval 5m
 ```
 
-Since backtesting knows it needs 100 candles to generate valid buy-signals, it'll load data from `20190101 - (100 * 5m)` - which is ~2019-12-31 15:30:00.
-If this data is available, Indicators will be calculated with this extended timerange. The startup period (Up to 2019-01-01 00:00:00) will then be removed before starting backtesting.
+Assuming `startup_candle_count` is set to 100, backtesting knows it needs 100 candles to generate valid buy signals. It will load data from `20190101 - (100 * 5m)` - which is ~2019-12-31 15:30:00.
+If this data is available, indicators will be calculated with this extended timerange. The instable startup period (up to 2019-01-01 00:00:00) will then be removed before starting backtesting.
 
 !!! Note
-    If data for the startup-period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-01 08:30:00.
+    If data for the startup period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-01 08:30:00.
 
 ### Buy signal rules
 
diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py
index df5c937cf..156f0e1e2 100644
--- a/freqtrade/configuration/timerange.py
+++ b/freqtrade/configuration/timerange.py
@@ -52,7 +52,7 @@ class TimeRange:
         """
         if (not self.starttype or (startup_candles
                                    and min_date.timestamp >= self.startts)):
-            # If no startts was defined, or test-data starts at the defined test-date
+            # If no startts was defined, or backtest-data starts at the defined backtest-date
             logger.warning("Moving start-date by %s candles to account for startup time.",
                            startup_candles)
             self.startts = (min_date.timestamp + ticker_interval_secs * startup_candles)
diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py
index c07b58da2..412b086c0 100644
--- a/freqtrade/data/history.py
+++ b/freqtrade/data/history.py
@@ -147,7 +147,7 @@ def load_pair_history(pair: str,
     """
 
     timerange_startup = deepcopy(timerange)
-    if startup_candles and timerange_startup:
+    if startup_candles > 0 and timerange_startup:
         logger.info('Using indicator startup period: %s ...', startup_candles)
         timerange_startup.subtract_start(timeframe_to_seconds(ticker_interval) * startup_candles)
 
diff --git a/tests/test_timerange.py b/tests/test_timerange.py
index d758092ed..5c35535f0 100644
--- a/tests/test_timerange.py
+++ b/tests/test_timerange.py
@@ -55,7 +55,7 @@ def test_adjust_start_if_necessary():
     assert x.startts == 1510694100 + (20 * 300)
 
     x = TimeRange('date', 'date', 1510700100, 1510780500)
-    # Do nothing, startupe is set and different min_date
+    # Do nothing, startup is set and different min_date
     x.adjust_start_if_necessary(300, 20, min_date)
     assert x.startts == 1510694100 + (20 * 300)