Merge branch 'develop' into feat/short

freqtrade/optimize/backtesting.py

@@ -12,20 +12,22 @@ from typing import Any, Dict, List, Optional, Tuple
 from numpy import nan
 from pandas import DataFrame
 
+from freqtrade import constants
 from freqtrade.configuration import TimeRange, validate_config_consistency
 from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data import history
-from freqtrade.data.btanalysis import trade_list_to_dataframe
+from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_to_dataframe
 from freqtrade.data.converter import trim_dataframe, trim_dataframes
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.enums import BacktestState, CandleType, SellType, TradingMode
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
+from freqtrade.misc import get_strategy_run_id
 from freqtrade.mixins import LoggingMixin
 from freqtrade.optimize.bt_progress import BTProgress
 from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
                                                  store_backtest_stats)
-from freqtrade.persistence import LocalTrade, PairLocks, Trade
+from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -63,9 +65,10 @@ class Backtesting:
 
         LoggingMixin.show_output = False
         self.config = config
-        self.results: Optional[Dict[str, Any]] = None
+        self.results: Dict[str, Any] = {}
 
         config['dry_run'] = True
+        self.run_ids: Dict[str, str] = {}
         self.strategylist: List[IStrategy] = []
         self.all_results: Dict[str, Dict] = {}
         self._exchange_name = self.config['exchange']['name']
@@ -373,12 +376,37 @@ class Backtesting:
         else:
             return sell_row[OPEN_IDX]
 
+    def _get_adjust_trade_entry_for_candle(self, trade: LocalTrade, row: Tuple
+                                           ) -> LocalTrade:
+
+        current_profit = trade.calc_profit_ratio(row[OPEN_IDX])
+        min_stake = self.exchange.get_min_pair_stake_amount(trade.pair, row[OPEN_IDX], -0.1)
+        max_stake = self.wallets.get_available_stake_amount()
+        stake_amount = strategy_safe_wrapper(self.strategy.adjust_trade_position,
+                                             default_retval=None)(
+            trade=trade, current_time=row[DATE_IDX].to_pydatetime(), current_rate=row[OPEN_IDX],
+            current_profit=current_profit, min_stake=min_stake, max_stake=max_stake)
+
+        # Check if we should increase our position
+        if stake_amount is not None and stake_amount > 0.0:
+
+            pos_trade = self._enter_trade(
+                trade.pair, row, 'short' if trade.is_short else 'long', stake_amount, trade)
+            if pos_trade is not None:
+                return pos_trade
+
+        return trade
+
     def _get_sell_trade_entry_for_candle(self, trade: LocalTrade,
                                          sell_row: Tuple) -> Optional[LocalTrade]:
         # TODO-lev: add interest / funding fees to trade object ->
         # Must be done either here, or one level higher ->
         # (if we don't want to do it at "detail" level)
 
+        # Check if we need to adjust our current positions
+        if self.strategy.position_adjustment_enable:
+            trade = self._get_adjust_trade_entry_for_candle(trade, sell_row)
+
         sell_candle_time = sell_row[DATE_IDX].to_pydatetime()
         enter = sell_row[SHORT_IDX] if trade.is_short else sell_row[LONG_IDX]
         exit_ = sell_row[ESHORT_IDX] if trade.is_short else sell_row[ELONG_IDX]
@@ -462,17 +490,14 @@ class Backtesting:
         else:
             return self._get_sell_trade_entry_for_candle(trade, sell_row)
 
-    def _enter_trade(self, pair: str, row: List, direction: str) -> Optional[LocalTrade]:
-        try:
-            stake_amount = self.wallets.get_trade_stake_amount(pair, None)
-        except DependencyException:
-            return None
+    def _enter_trade(self, pair: str, row: Tuple, direction: str,
+                     stake_amount: Optional[float] = None,
+                     trade: Optional[LocalTrade] = None) -> Optional[LocalTrade]:
         current_time = row[DATE_IDX].to_pydatetime()
-
         # let's call the custom entry price, using the open price as default price
         propose_rate = strategy_safe_wrapper(self.strategy.custom_entry_price,
                                              default_retval=row[OPEN_IDX])(
-            pair=pair, current_time=row[DATE_IDX].to_pydatetime(),
+            pair=pair, current_time=current_time,
             proposed_rate=row[OPEN_IDX])  # default value is the open rate
 
         # Move rate to within the candle's low/high rate
@@ -481,15 +506,25 @@ class Backtesting:
         min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, propose_rate, -0.05) or 0
         max_stake_amount = self.wallets.get_available_stake_amount()
 
-        stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount,
-                                             default_retval=stake_amount)(
-            pair=pair, current_time=current_time, current_rate=propose_rate,
-            proposed_stake=stake_amount, min_stake=min_stake_amount, max_stake=max_stake_amount,
-            side=direction)
+        pos_adjust = trade is not None
+        if not pos_adjust:
+            try:
+                stake_amount = self.wallets.get_trade_stake_amount(pair, None)
+            except DependencyException:
+                return trade
+
+            stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount,
+                                                 default_retval=stake_amount)(
+                pair=pair, current_time=current_time, current_rate=propose_rate,
+                proposed_stake=stake_amount, min_stake=min_stake_amount, max_stake=max_stake_amount,
+                side=direction)
+
         stake_amount = self.wallets.validate_stake_amount(pair, stake_amount, min_stake_amount)
 
         if not stake_amount:
-            return None
+            # In case of pos adjust, still return the original trade
+            # If not pos adjust, trade is None
+            return trade
 
         max_leverage = self.exchange.get_max_leverage(pair, stake_amount)
         leverage = strategy_safe_wrapper(self.strategy.leverage, default_retval=1.0)(
@@ -506,31 +541,54 @@ class Backtesting:
         order_type = self.strategy.order_types['buy']
         time_in_force = self.strategy.order_time_in_force['sell']
         # Confirm trade entry:
-        if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)(
-                pair=pair, order_type=order_type, amount=stake_amount, rate=propose_rate,
-                time_in_force=time_in_force, current_time=current_time,
-                side=direction):
-            return None
+        if not pos_adjust:
+            if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)(
+                    pair=pair, order_type=order_type, amount=stake_amount, rate=propose_rate,
+                    time_in_force=time_in_force, current_time=current_time,
+                    side=direction):
+                return None
 
         if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount):
-            # Enter trade
-            has_enter_tag = len(row) >= ENTER_TAG_IDX + 1
-            trade = LocalTrade(
-                pair=pair,
-                open_rate=row[OPEN_IDX],
-                open_date=current_time,
-                stake_amount=stake_amount,
-                amount=round((stake_amount / propose_rate) * leverage, 8),
-                fee_open=self.fee,
-                fee_close=self.fee,
-                is_open=True,
-                enter_tag=row[ENTER_TAG_IDX] if has_enter_tag else None,
-                exchange=self._exchange_name,
-                is_short=(direction == 'short'),
-                leverage=leverage,
+            amount = round(stake_amount / propose_rate, 8)
+            if trade is None:
+                # Enter trade
+                has_buy_tag = len(row) >= ENTER_TAG_IDX + 1
+                trade = LocalTrade(
+                    pair=pair,
+                    open_rate=propose_rate,
+                    open_date=current_time,
+                    stake_amount=stake_amount,
+                    amount=round((stake_amount / propose_rate) * leverage, 8),
+                    fee_open=self.fee,
+                    fee_close=self.fee,
+                    is_open=True,
+                    enter_tag=row[ENTER_TAG_IDX] if has_buy_tag else None,
+                    exchange=self._exchange_name,
+                    is_short=(direction == 'short'),
+                    leverage=leverage,
+                    orders=[]
+                )
+            trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True)
+
+            order = Order(
+                ft_is_open=False,
+                ft_pair=trade.pair,
+                symbol=trade.pair,
+                ft_order_side="buy",
+                side="buy",
+                order_type="market",
+                status="closed",
+                price=propose_rate,
+                average=propose_rate,
+                amount=amount,
+                filled=amount,
+                cost=stake_amount + trade.fee_open
             )
-            return trade
-        return None
+            trade.orders.append(order)
+            if pos_adjust:
+                trade.recalc_trade_from_orders()
+
+        return trade
 
     def handle_left_open(self, open_trades: Dict[str, List[LocalTrade]],
                          data: Dict[str, List[Tuple]]) -> List[LocalTrade]:
@@ -734,6 +792,7 @@ class Backtesting:
         )
         backtest_end_time = datetime.now(timezone.utc)
         results.update({
+            'run_id': self.run_ids.get(strat.get_strategy_name(), ''),
             'backtest_start_time': int(backtest_start_time.timestamp()),
             'backtest_end_time': int(backtest_end_time.timestamp()),
         })
@@ -741,6 +800,33 @@ class Backtesting:
 
         return min_date, max_date
 
+    def _get_min_cached_backtest_date(self):
+        min_backtest_date = None
+        backtest_cache_age = self.config.get('backtest_cache', constants.BACKTEST_CACHE_DEFAULT)
+        if self.timerange.stopts == 0 or datetime.fromtimestamp(
+           self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc):
+            logger.warning('Backtest result caching disabled due to use of open-ended timerange.')
+        elif backtest_cache_age == 'day':
+            min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(days=1)
+        elif backtest_cache_age == 'week':
+            min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(weeks=1)
+        elif backtest_cache_age == 'month':
+            min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(weeks=4)
+        return min_backtest_date
+
+    def load_prior_backtest(self):
+        self.run_ids = {
+            strategy.get_strategy_name(): get_strategy_run_id(strategy)
+            for strategy in self.strategylist
+        }
+
+        # Load previous result that will be updated incrementally.
+        # This can be circumvented in certain instances in combination with downloading more data
+        min_backtest_date = self._get_min_cached_backtest_date()
+        if min_backtest_date is not None:
+            self.results = find_existing_backtest_stats(
+                self.config['user_data_dir'] / 'backtest_results', self.run_ids, min_backtest_date)
+
     def start(self) -> None:
         """
         Run backtesting end-to-end
@@ -752,15 +838,38 @@ class Backtesting:
         self.load_bt_data_detail()
         logger.info("Dataload complete. Calculating indicators")
 
-        for strat in self.strategylist:
-            min_date, max_date = self.backtest_one_strategy(strat, data, timerange)
-        if len(self.strategylist) > 0:
+        self.load_prior_backtest()
 
-            self.results = generate_backtest_stats(data, self.all_results,
-                                                   min_date=min_date, max_date=max_date)
+        for strat in self.strategylist:
+            if self.results and strat.get_strategy_name() in self.results['strategy']:
+                # When previous result hash matches - reuse that result and skip backtesting.
+                logger.info(f'Reusing result of previous backtest for {strat.get_strategy_name()}')
+                continue
+            min_date, max_date = self.backtest_one_strategy(strat, data, timerange)
+
+        # Update old results with new ones.
+        if len(self.all_results) > 0:
+            results = generate_backtest_stats(
+                data, self.all_results, min_date=min_date, max_date=max_date)
+            if self.results:
+                self.results['metadata'].update(results['metadata'])
+                self.results['strategy'].update(results['strategy'])
+                self.results['strategy_comparison'].extend(results['strategy_comparison'])
+            else:
+                self.results = results
 
             if self.config.get('export', 'none') == 'trades':
                 store_backtest_stats(self.config['exportfilename'], self.results)
 
+        # Results may be mixed up now. Sort them so they follow --strategy-list order.
+        if 'strategy_list' in self.config and len(self.results) > 0:
+            self.results['strategy_comparison'] = sorted(
+                self.results['strategy_comparison'],
+                key=lambda c: self.config['strategy_list'].index(c['key']))
+            self.results['strategy'] = dict(
+                sorted(self.results['strategy'].items(),
+                       key=lambda kv: self.config['strategy_list'].index(kv[0])))
+
+        if len(self.strategylist) > 0:
             # Show backtest results
             show_backtest_results(self.config, self.results)

freqtrade/optimize/edge_cli.py

@@ -34,7 +34,7 @@ class EdgeCli:
         self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT
         self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
         self.strategy = StrategyResolver.load_strategy(self.config)
-        self.strategy.dp = DataProvider(config, None)
+        self.strategy.dp = DataProvider(config, self.exchange)
 
         validate_config_consistency(self.config)
 

freqtrade/optimize/hyperopt_tools.py

@@ -137,6 +137,7 @@ class HyperoptTools():
         }
         if not HyperoptTools._test_hyperopt_results_exist(results_file):
             # No file found.
+            logger.warning(f"Hyperopt file {results_file} not found.")
             return [], 0
 
         epochs = []

freqtrade/optimize/optimize_reports.py

@@ -11,7 +11,8 @@ from tabulate import tabulate
 from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT
 from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change,
                                        calculate_max_drawdown)
-from freqtrade.misc import decimals_per_coin, file_dump_json, round_coin_value
+from freqtrade.misc import (decimals_per_coin, file_dump_json, get_backtest_metadata_filename,
+                            round_coin_value)
 
 
 logger = logging.getLogger(__name__)
@@ -33,6 +34,11 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N
             recordfilename.parent,
             f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}'
         ).with_suffix(recordfilename.suffix)
+
+    # Store metadata separately.
+    file_dump_json(get_backtest_metadata_filename(filename), stats['metadata'])
+    del stats['metadata']
+
     file_dump_json(filename, stats)
 
     latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN)
@@ -515,16 +521,26 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
     :param max_date: Backtest end date
     :return: Dictionary containing results per strategy and a strategy summary.
     """
-    result: Dict[str, Any] = {'strategy': {}}
+    result: Dict[str, Any] = {
+        'metadata': {},
+        'strategy': {},
+        'strategy_comparison': [],
+    }
     market_change = calculate_market_change(btdata, 'close')
+    metadata = {}
     pairlist = list(btdata.keys())
     for strategy, content in all_results.items():
         strat_stats = generate_strategy_stats(pairlist, strategy, content,
                                               min_date, max_date, market_change=market_change)
+        metadata[strategy] = {
+            'run_id': content['run_id'],
+            'backtest_start_time': content['backtest_start_time'],
+        }
         result['strategy'][strategy] = strat_stats
 
     strategy_results = generate_strategy_comparison(bt_stats=result['strategy'])
 
+    result['metadata'] = metadata
     result['strategy_comparison'] = strategy_results
 
     return result