Merge pull request #6543 from froggleston/v3_fixes

Add support for storing buy candle indicator rows in backtesting results

freqtrade/optimize/backtesting.py

@@ -19,13 +19,15 @@ from freqtrade.data import history
 from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_to_dataframe
 from freqtrade.data.converter import trim_dataframe, trim_dataframes
 from freqtrade.data.dataprovider import DataProvider
-from freqtrade.enums import BacktestState, CandleType, ExitCheckTuple, ExitType, TradingMode
+from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType, RunMode,
+                             TradingMode)
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
 from freqtrade.misc import get_strategy_run_id
 from freqtrade.mixins import LoggingMixin
 from freqtrade.optimize.bt_progress import BTProgress
 from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
+                                                 store_backtest_signal_candles,
                                                  store_backtest_stats)
 from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade
 from freqtrade.plugins.pairlistmanager import PairListManager
@@ -73,6 +75,8 @@ class Backtesting:
         self.run_ids: Dict[str, str] = {}
         self.strategylist: List[IStrategy] = []
         self.all_results: Dict[str, Dict] = {}
+        self.processed_dfs: Dict[str, Dict] = {}
+
         self._exchange_name = self.config['exchange']['name']
         self.exchange = ExchangeResolver.load_exchange(self._exchange_name, self.config)
         self.dataprovider = DataProvider(self.config, self.exchange)
@@ -1070,8 +1074,31 @@ class Backtesting:
         })
         self.all_results[self.strategy.get_strategy_name()] = results
 
+        if (self.config.get('export', 'none') == 'signals' and
+                self.dataprovider.runmode == RunMode.BACKTEST):
+            self._generate_trade_signal_candles(preprocessed_tmp, results)
+
         return min_date, max_date
 
+    def _generate_trade_signal_candles(self, preprocessed_df, bt_results):
+        signal_candles_only = {}
+        for pair in preprocessed_df.keys():
+            signal_candles_only_df = DataFrame()
+
+            pairdf = preprocessed_df[pair]
+            resdf = bt_results['results']
+            pairresults = resdf.loc[(resdf["pair"] == pair)]
+
+            if pairdf.shape[0] > 0:
+                for t, v in pairresults.open_date.items():
+                    allinds = pairdf.loc[(pairdf['date'] < v)]
+                    signal_inds = allinds.iloc[[-1]]
+                    signal_candles_only_df = signal_candles_only_df.append(signal_inds)
+
+                signal_candles_only[pair] = signal_candles_only_df
+
+        self.processed_dfs[self.strategy.get_strategy_name()] = signal_candles_only
+
     def _get_min_cached_backtest_date(self):
         min_backtest_date = None
         backtest_cache_age = self.config.get('backtest_cache', constants.BACKTEST_CACHE_DEFAULT)
@@ -1130,9 +1157,13 @@ class Backtesting:
             else:
                 self.results = results
 
-            if self.config.get('export', 'none') == 'trades':
+            if self.config.get('export', 'none') in ('trades', 'signals'):
                 store_backtest_stats(self.config['exportfilename'], self.results)
 
+            if (self.config.get('export', 'none') == 'signals' and
+                    self.dataprovider.runmode == RunMode.BACKTEST):
+                store_backtest_signal_candles(self.config['exportfilename'], self.processed_dfs)
+
         # Results may be mixed up now. Sort them so they follow --strategy-list order.
         if 'strategy_list' in self.config and len(self.results) > 0:
             self.results['strategy_comparison'] = sorted(

freqtrade/optimize/optimize_reports.py

@@ -11,8 +11,8 @@ from tabulate import tabulate
 from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT
 from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change,
                                        calculate_max_drawdown)
-from freqtrade.misc import (decimals_per_coin, file_dump_json, get_backtest_metadata_filename,
-                            round_coin_value)
+from freqtrade.misc import (decimals_per_coin, file_dump_joblib, file_dump_json,
+                            get_backtest_metadata_filename, round_coin_value)
 
 
 logger = logging.getLogger(__name__)
@@ -45,6 +45,29 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N
     file_dump_json(latest_filename, {'latest_backtest': str(filename.name)})
 
 
+def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict]) -> Path:
+    """
+    Stores backtest trade signal candles
+    :param recordfilename: Path object, which can either be a filename or a directory.
+        Filenames will be appended with a timestamp right before the suffix
+        while for directories, <directory>/backtest-result-<datetime>_signals.pkl will be used
+        as filename
+    :param stats: Dict containing the backtesting signal candles
+    """
+    if recordfilename.is_dir():
+        filename = (recordfilename /
+                    f'backtest-result-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals.pkl')
+    else:
+        filename = Path.joinpath(
+            recordfilename.parent,
+            f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals.pkl'
+        )
+
+    file_dump_joblib(filename, candles)
+
+    return filename
+
+
 def _get_line_floatfmt(stake_currency: str) -> List[str]:
     """
     Generate floatformat (goes in line with _generate_result_line())