Refactor loading of bt data to backtesting ...

freqtrade/data/history.py

@@ -183,6 +183,7 @@ def load_data(datadir: Path,
               timerange: Optional[TimeRange] = None,
               fill_up_missing: bool = True,
               startup_candles: int = 0,
+              fail_without_data: bool = False
               ) -> Dict[str, DataFrame]:
     """
     Loads ticker history data for a list of pairs
@@ -195,6 +196,7 @@ def load_data(datadir: Path,
     :param timerange: Limit data to be loaded to this timerange
     :param fill_up_missing: Fill missing values with "No action"-candles
     :param startup_candles: Additional candles to load at the start of the period
+    :param fail_without_data: Raise OperationalException if no data is found.
     :return: dict(<pair>:<Dataframe>)
     TODO: refresh_pairs is still used by edge to keep the data uptodate.
         This should be replaced in the future. Instead, writing the current candles to disk
@@ -208,9 +210,13 @@ def load_data(datadir: Path,
                                  datadir=datadir, timerange=timerange,
                                  refresh_pairs=refresh_pairs,
                                  exchange=exchange,
-                                 fill_up_missing=fill_up_missing)
+                                 fill_up_missing=fill_up_missing,
+                                 startup_candles=startup_candles)
         if hist is not None:
             result[pair] = hist
+
+    if fail_without_data and not result:
+        raise OperationalException("No data found. Terminating.")
     return result
 
 

freqtrade/optimize/backtesting.py

@@ -105,6 +105,31 @@ class Backtesting:
         # And the regular "stoploss" function would not apply to that case
         self.strategy.order_types['stoploss_on_exchange'] = False
 
+    def load_bt_data(self):
+        timerange = TimeRange.parse_timerange(None if self.config.get(
+            'timerange') is None else str(self.config.get('timerange')))
+
+        data = history.load_data(
+            datadir=Path(self.config['datadir']),
+            pairs=self.config['exchange']['pair_whitelist'],
+            ticker_interval=self.ticker_interval,
+            timerange=timerange,
+            startup_candles=self.required_startup,
+            fail_without_data=True,
+        )
+
+        min_date, max_date = history.get_timeframe(data)
+
+        logger.info(
+            'Loading data from %s up to %s (%s days)..',
+            min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
+        )
+        # Adjust startts forward if not enough data is available
+        timerange.adjust_start_if_necessary(timeframe_to_seconds(self.ticker_interval),
+                                            self.required_startup, min_date)
+
+        return data, timerange
+
     def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame,
                              skip_nan: bool = False) -> str:
         """
@@ -414,42 +439,18 @@ class Backtesting:
         :return: None
         """
         data: Dict[str, Any] = {}
-        pairs = self.config['exchange']['pair_whitelist']
         logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
         logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
-
-        timerange = TimeRange.parse_timerange(None if self.config.get(
-            'timerange') is None else str(self.config.get('timerange')))
-
-        data = history.load_data(
-            datadir=Path(self.config['datadir']),
-            pairs=pairs,
-            ticker_interval=self.ticker_interval,
-            timerange=timerange,
-            startup_candles=self.required_startup
-        )
-
-        if not data:
-            logger.critical("No data found. Terminating.")
-            return
         # Use max_open_trades in backtesting, except --disable-max-market-positions is set
         if self.config.get('use_max_market_positions', True):
             max_open_trades = self.config['max_open_trades']
         else:
             logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...')
             max_open_trades = 0
+
+        data, timerange = self.load_bt_data()
+
         all_results = {}
-
-        min_date, max_date = history.get_timeframe(data)
-
-        logger.info(
-            'Loading backtest data from %s up to %s (%s days)..',
-            min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
-        )
-        # Adjust startts forward if not enough data is available
-        timerange.adjust_start_if_necessary(timeframe_to_seconds(self.ticker_interval),
-                                            self.required_startup, min_date)
-
         for strat in self.strategylist:
             logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
             self._set_strategy(strat)

freqtrade/optimize/hyperopt.py

@@ -23,7 +23,7 @@ from skopt import Optimizer
 from skopt.space import Dimension
 
 from freqtrade.configuration import TimeRange
-from freqtrade.data.history import load_data, get_timeframe
+from freqtrade.data.history import load_data, get_timeframe, trim_dataframe
 from freqtrade.misc import round_dict
 from freqtrade.optimize.backtesting import Backtesting
 # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
@@ -379,30 +379,19 @@ class Hyperopt:
             )
 
     def start(self) -> None:
-        timerange = TimeRange.parse_timerange(None if self.config.get(
+        data, timerange = self.backtesting.load_bt_data()
-            'timerange') is None else str(self.config.get('timerange')))
-        data = load_data(
-            datadir=Path(self.config['datadir']),
-            pairs=self.config['exchange']['pair_whitelist'],
-            ticker_interval=self.backtesting.ticker_interval,
-            timerange=timerange
-        )
 
-        if not data:
+        preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data)
-            logger.critical("No data found. Terminating.")
-            return
 
+        # Trim startup period from analyzed dataframe
+        for pair, df in preprocessed.items():
+            preprocessed[pair] = trim_dataframe(df, timerange)
         min_date, max_date = get_timeframe(data)
 
         logger.info(
             'Hyperopting with data from %s up to %s (%s days)..',
-            min_date.isoformat(),
+            min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
-            max_date.isoformat(),
-            (max_date - min_date).days
         )
-
-        preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data)
-
         dump(preprocessed, self.tickerdata_pickle)
 
         # We don't need exchange instance anymore while running hyperopt