Merge branch 'develop' into pairlocks_direction

freqtrade/optimize/backtest_caching.py

@@ -0,0 +1,40 @@
+import hashlib
+from copy import deepcopy
+from pathlib import Path
+from typing import Union
+
+import rapidjson
+
+
+def get_strategy_run_id(strategy) -> str:
+    """
+    Generate unique identification hash for a backtest run. Identical config and strategy file will
+    always return an identical hash.
+    :param strategy: strategy object.
+    :return: hex string id.
+    """
+    digest = hashlib.sha1()
+    config = deepcopy(strategy.config)
+
+    # Options that have no impact on results of individual backtest.
+    not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server')
+    for k in not_important_keys:
+        if k in config:
+            del config[k]
+
+    # Explicitly allow NaN values (e.g. max_open_trades).
+    # as it does not matter for getting the hash.
+    digest.update(rapidjson.dumps(config, default=str,
+                                  number_mode=rapidjson.NM_NAN).encode('utf-8'))
+    # Include _ft_params_from_file - so changing parameter files cause cache eviction
+    digest.update(rapidjson.dumps(
+        strategy._ft_params_from_file, default=str, number_mode=rapidjson.NM_NAN).encode('utf-8'))
+    with open(strategy.__file__, 'rb') as fp:
+        digest.update(fp.read())
+    return digest.hexdigest().lower()
+
+
+def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path:
+    """Return metadata filename for specified backtest results file."""
+    filename = Path(filename)
+    return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}')

freqtrade/optimize/backtesting.py

@@ -9,6 +9,7 @@ from copy import deepcopy
 from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, Optional, Tuple
 
+import pandas as pd
 from numpy import nan
 from pandas import DataFrame
 
@@ -23,8 +24,8 @@ from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType
                              TradingMode)
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
-from freqtrade.misc import get_strategy_run_id
 from freqtrade.mixins import LoggingMixin
+from freqtrade.optimize.backtest_caching import get_strategy_run_id
 from freqtrade.optimize.bt_progress import BTProgress
 from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
                                                  store_backtest_signal_candles,
@@ -53,6 +54,11 @@ ESHORT_IDX = 8  # Exit short
 ENTER_TAG_IDX = 9
 EXIT_TAG_IDX = 10
 
+# Every change to this headers list must evaluate further usages of the resulting tuple
+# and eventually change the constants for indexes at the top
+HEADERS = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long',
+           'enter_short', 'exit_short', 'enter_tag', 'exit_tag']
+
 
 class Backtesting:
     """
@@ -181,6 +187,7 @@ class Backtesting:
         # since a "perfect" stoploss-exit is assumed anyway
         # And the regular "stoploss" function would not apply to that case
         self.strategy.order_types['stoploss_on_exchange'] = False
+        self.strategy.bot_start()
 
     def _load_protections(self, strategy: IStrategy):
         if self.config.get('enable_protections', False):
@@ -263,10 +270,18 @@ class Backtesting:
                 candle_type=CandleType.from_string(self.exchange._ft_has["mark_ohlcv_price"])
             )
             # Combine data to avoid combining the data per trade.
+            unavailable_pairs = []
             for pair in self.pairlists.whitelist:
+                if pair not in self.exchange._leverage_tiers:
+                    unavailable_pairs.append(pair)
+                    continue
                 self.futures_data[pair] = funding_rates_dict[pair].merge(
                     mark_rates_dict[pair], on='date', how="inner", suffixes=["_fund", "_mark"])
 
+            if unavailable_pairs:
+                raise OperationalException(
+                        f"Pairs {', '.join(unavailable_pairs)} got no leverage tiers available. "
+                        "It is therefore impossible to backtest with this pair at the moment.")
         else:
             self.futures_data = {}
 
@@ -304,10 +319,7 @@ class Backtesting:
         :param processed: a processed dictionary with format {pair, data}, which gets cleared to
         optimize memory usage!
         """
-        # Every change to this headers list must evaluate further usages of the resulting tuple
-        # and eventually change the constants for indexes at the top
-        headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long',
-                   'enter_short', 'exit_short', 'enter_tag', 'exit_tag']
+
         data: Dict = {}
         self.progress.init_step(BacktestState.CONVERT, len(processed))
 
@@ -319,7 +331,7 @@ class Backtesting:
 
             if not pair_data.empty:
                 # Cleanup from prior runs
-                pair_data.drop(headers[5:] + ['buy', 'sell'], axis=1, errors='ignore')
+                pair_data.drop(HEADERS[5:] + ['buy', 'sell'], axis=1, errors='ignore')
 
             df_analyzed = self.strategy.advise_exit(
                 self.strategy.advise_entry(pair_data, {'pair': pair}),
@@ -338,7 +350,7 @@ class Backtesting:
 
             # To avoid using data from future, we use entry/exit signals shifted
             # from the previous candle
-            for col in headers[5:]:
+            for col in HEADERS[5:]:
                 tag_col = col in ('enter_tag', 'exit_tag')
                 if col in df_analyzed.columns:
                     df_analyzed.loc[:, col] = df_analyzed.loc[:, col].replace(
@@ -350,7 +362,7 @@ class Backtesting:
 
             # Convert from Pandas to list for performance reasons
             # (Looping Pandas is slow.)
-            data[pair] = df_analyzed[headers].values.tolist() if not df_analyzed.empty else []
+            data[pair] = df_analyzed[HEADERS].values.tolist() if not df_analyzed.empty else []
         return data
 
     def _get_close_rate(self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple,
@@ -514,10 +526,10 @@ class Backtesting:
 
         exit_candle_time: datetime = row[DATE_IDX].to_pydatetime()
         enter = row[SHORT_IDX] if trade.is_short else row[LONG_IDX]
-        exit_ = row[ESHORT_IDX] if trade.is_short else row[ELONG_IDX]
+        exit_sig = row[ESHORT_IDX] if trade.is_short else row[ELONG_IDX]
         exit_ = self.strategy.should_exit(
             trade, row[OPEN_IDX], exit_candle_time,  # type: ignore
-            enter=enter, exit_=exit_,
+            enter=enter, exit_=exit_sig,
             low=row[LOW_IDX], high=row[HIGH_IDX]
         )
 
@@ -539,7 +551,8 @@ class Backtesting:
                                                       default_retval=closerate)(
                         pair=trade.pair, trade=trade,
                         current_time=exit_candle_time,
-                        proposed_rate=closerate, current_profit=current_profit)
+                        proposed_rate=closerate, current_profit=current_profit,
+                        exit_tag=exit_.exit_reason)
                     # We can't place orders lower than current low.
                     # freqtrade does not support this in live, and the order would fill immediately
                     if trade.is_short:
@@ -566,6 +579,7 @@ class Backtesting:
                 len(row) > EXIT_TAG_IDX
                 and row[EXIT_TAG_IDX] is not None
                 and len(row[EXIT_TAG_IDX]) > 0
+                and exit_.exit_type in (ExitType.EXIT_SIGNAL,)
             ):
                 trade.exit_reason = row[EXIT_TAG_IDX]
 
@@ -624,9 +638,7 @@ class Backtesting:
             detail_data.loc[:, 'exit_short'] = row[ESHORT_IDX]
             detail_data.loc[:, 'enter_tag'] = row[ENTER_TAG_IDX]
             detail_data.loc[:, 'exit_tag'] = row[EXIT_TAG_IDX]
-            headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long',
-                       'enter_short', 'exit_short', 'enter_tag', 'exit_tag']
-            for det_row in detail_data[headers].values.tolist():
+            for det_row in detail_data[HEADERS].values.tolist():
                 res = self._get_exit_trade_entry_for_candle(trade, det_row)
                 if res:
                     return res
@@ -1028,7 +1040,7 @@ class Backtesting:
                               timerange: TimeRange):
         self.progress.init_step(BacktestState.ANALYZE, 0)
 
-        logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
+        logger.info(f"Running backtesting for Strategy {strat.get_strategy_name()}")
         backtest_start_time = datetime.now(timezone.utc)
         self._set_strategy(strat)
 
@@ -1095,7 +1107,7 @@ class Backtesting:
                 for t, v in pairresults.open_date.items():
                     allinds = pairdf.loc[(pairdf['date'] < v)]
                     signal_inds = allinds.iloc[[-1]]
-                    signal_candles_only_df = signal_candles_only_df.append(signal_inds)
+                    signal_candles_only_df = pd.concat([signal_candles_only_df, signal_inds])
 
                 signal_candles_only[pair] = signal_candles_only_df
 

freqtrade/optimize/edge_cli.py

@@ -44,6 +44,7 @@ class EdgeCli:
 
         self.edge._timerange = TimeRange.parse_timerange(None if self.config.get(
             'timerange') is None else str(self.config.get('timerange')))
+        self.strategy.bot_start()
 
     def start(self) -> None:
         result = self.edge.calculate(self.config['exchange']['pair_whitelist'])

freqtrade/optimize/hyperopt.py

@@ -468,6 +468,7 @@ class Hyperopt:
         self.backtesting.exchange._api = None
         self.backtesting.exchange._api_async = None
         self.backtesting.exchange.loop = None  # type: ignore
+        self.backtesting.exchange._loop_lock = None  # type: ignore
         # self.backtesting.exchange = None  # type: ignore
         self.backtesting.pairlists = None  # type: ignore
 

freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py

@@ -10,7 +10,7 @@ from typing import Any, Dict
 
 from pandas import DataFrame
 
-from freqtrade.data.btanalysis import calculate_max_drawdown
+from freqtrade.data.metrics import calculate_max_drawdown
 from freqtrade.optimize.hyperopt import IHyperOptLoss
 
 

freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py

@@ -8,7 +8,7 @@ from datetime import datetime
 
 from pandas import DataFrame
 
-from freqtrade.data.btanalysis import calculate_max_drawdown
+from freqtrade.data.metrics import calculate_max_drawdown
 from freqtrade.optimize.hyperopt import IHyperOptLoss
 
 

freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py

@@ -9,7 +9,7 @@ individual needs.
 """
 from pandas import DataFrame
 
-from freqtrade.data.btanalysis import calculate_max_drawdown
+from freqtrade.data.metrics import calculate_max_drawdown
 from freqtrade.optimize.hyperopt import IHyperOptLoss
 
 

freqtrade/optimize/optimize_reports.py

@@ -9,10 +9,10 @@ from pandas import DataFrame, to_datetime
 from tabulate import tabulate
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT
-from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change,
-                                       calculate_max_drawdown)
-from freqtrade.misc import (decimals_per_coin, file_dump_joblib, file_dump_json,
-                            get_backtest_metadata_filename, round_coin_value)
+from freqtrade.data.metrics import (calculate_cagr, calculate_csum, calculate_market_change,
+                                    calculate_max_drawdown)
+from freqtrade.misc import decimals_per_coin, file_dump_joblib, file_dump_json, round_coin_value
+from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename
 
 
 logger = logging.getLogger(__name__)
@@ -446,6 +446,7 @@ def generate_strategy_stats(pairlist: List[str],
         'profit_total_abs': results['profit_abs'].sum(),
         'profit_total_long_abs': results.loc[~results['is_short'], 'profit_abs'].sum(),
         'profit_total_short_abs': results.loc[results['is_short'], 'profit_abs'].sum(),
+        'cagr': calculate_cagr(backtest_days, start_balance, content['final_balance']),
         'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT),
         'backtest_start_ts': int(min_date.timestamp() * 1000),
         'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT),
@@ -746,6 +747,7 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'],
                                                   strat_results['stake_currency'])),
             ('Total profit %', f"{strat_results['profit_total']:.2%}"),
+            ('CAGR %', f"{strat_results['cagr']:.2%}" if 'cagr' in strat_results else 'N/A'),
             ('Trades per day', strat_results['trades_per_day']),
             ('Avg. daily profit %',
              f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"),