Merge pull request #1290 from freqtrade/fix/backtest_toomanyopen

fix backtesting not respecting max_open_trades

freqtrade/optimize/backtesting.py

@@ -66,6 +66,7 @@ class Backtesting(object):
         if self.config.get('strategy_list', None):
             # Force one interval
             self.ticker_interval = str(self.config.get('ticker_interval'))
+            self.ticker_interval_mins = constants.TICKER_INTERVAL_MINUTES[self.ticker_interval]
             for strat in list(self.config['strategy_list']):
                 stratconf = deepcopy(self.config)
                 stratconf['strategy'] = strat
@@ -86,6 +87,8 @@ class Backtesting(object):
         """
         self.strategy = strategy
         self.ticker_interval = self.config.get('ticker_interval')
+        self.ticker_interval_mins = constants.TICKER_INTERVAL_MINUTES[self.ticker_interval]
+        self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe
         self.advise_buy = strategy.advise_buy
         self.advise_sell = strategy.advise_sell
 
@@ -280,8 +283,13 @@ class Backtesting(object):
         processed = args['processed']
         max_open_trades = args.get('max_open_trades', 0)
         position_stacking = args.get('position_stacking', False)
+        start_date = args['start_date']
+        end_date = args['end_date']
         trades = []
         trade_count_lock: Dict = {}
+        ticker: Dict = {}
+        pairs = []
+        # Create ticker dict
         for pair, pair_data in processed.items():
             pair_data['buy'], pair_data['sell'] = 0, 0  # cleanup from previous run
 
@@ -296,15 +304,28 @@ class Backtesting(object):
 
             # Convert from Pandas to list for performance reasons
             # (Looping Pandas is slow.)
-            ticker = [x for x in ticker_data.itertuples()]
+            ticker[pair] = [x for x in ticker_data.itertuples()]
+            pairs.append(pair)
+
+        lock_pair_until: Dict = {}
+        tmp = start_date + timedelta(minutes=self.ticker_interval_mins)
+        index = 0
+        # Loop timerange and test per pair
+        while tmp < end_date:
+            # print(f"time: {tmp}")
+            for i, pair in enumerate(ticker):
+                try:
+                    row = ticker[pair][index]
+                except IndexError:
+                    # missing Data for one pair ...
+                    # Warnings for this are shown by `validate_backtest_data`
+                    continue
 
-            lock_pair_until = None
-            for index, row in enumerate(ticker):
                 if row.buy == 0 or row.sell == 1:
                     continue  # skip rows where no buy signal or that would immediately sell off
 
                 if not position_stacking:
-                    if lock_pair_until is not None and row.date <= lock_pair_until:
+                    if pair in lock_pair_until and row.date <= lock_pair_until[pair]:
                         continue
                 if max_open_trades > 0:
                     # Check if max_open_trades has already been reached for the given date
@@ -313,17 +334,19 @@ class Backtesting(object):
 
                     trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
 
-                trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:],
+                trade_entry = self._get_sell_trade_entry(pair, row, ticker[pair][index + 1:],
                                                          trade_count_lock, args)
 
                 if trade_entry:
-                    lock_pair_until = trade_entry.close_time
+                    lock_pair_until[pair] = trade_entry.close_time
                     trades.append(trade_entry)
                 else:
                     # Set lock_pair_until to end of testing period if trade could not be closed
                     # This happens only if the buy-signal was with the last candle
-                    lock_pair_until = ticker_data.iloc[-1].date
+                    lock_pair_until[pair] = end_date
 
+            tmp += timedelta(minutes=self.ticker_interval_mins)
+            index += 1
         return DataFrame.from_records(trades, columns=BacktestResult._fields)
 
     def start(self) -> None:
@@ -390,6 +413,8 @@ class Backtesting(object):
                     'processed': preprocessed,
                     'max_open_trades': max_open_trades,
                     'position_stacking': self.config.get('position_stacking', False),
+                    'start_date': min_date,
+                    'end_date': max_date,
                 }
             )
 

freqtrade/optimize/hyperopt.py

@@ -20,7 +20,7 @@ from skopt.space import Dimension
 
 from freqtrade.arguments import Arguments
 from freqtrade.configuration import Configuration
-from freqtrade.optimize import load_data
+from freqtrade.optimize import load_data, get_timeframe
 from freqtrade.optimize.backtesting import Backtesting
 from freqtrade.resolvers import HyperOptResolver
 
@@ -167,11 +167,14 @@ class Hyperopt(Backtesting):
             self.strategy.stoploss = params['stoploss']
 
         processed = load(TICKERDATA_PICKLE)
+        min_date, max_date = get_timeframe(processed)
         results = self.backtest(
             {
                 'stake_amount': self.config['stake_amount'],
                 'processed': processed,
                 'position_stacking': self.config.get('position_stacking', True),
+                'start_date': min_date,
+                'end_date': max_date,
             }
         )
         result_explanation = self.format_results(results)

freqtrade/tests/optimize/__init__.py

@@ -4,9 +4,10 @@ import arrow
 from pandas import DataFrame
 
 from freqtrade.strategy.interface import SellType
+from freqtrade.constants import TICKER_INTERVAL_MINUTES
 
 ticker_start_time = arrow.get(2018, 10, 3)
-ticker_interval_in_minute = 60
+tests_ticker_interval = "1h"
 
 
 class BTrade(NamedTuple):
@@ -30,8 +31,8 @@ class BTContainer(NamedTuple):
 
 
 def _get_frame_time_from_offset(offset):
-    return ticker_start_time.shift(
+    return ticker_start_time.shift(minutes=(offset * TICKER_INTERVAL_MINUTES[tests_ticker_interval])
-        minutes=(offset * ticker_interval_in_minute)).datetime.replace(tzinfo=None)
+                                   ).datetime.replace(tzinfo=None)
 
 
 def _build_backtest_dataframe(ticker_with_signals):

freqtrade/tests/optimize/test_backtest_detail.py

@@ -6,10 +6,11 @@ from pandas import DataFrame
 import pytest
 
 
+from freqtrade.optimize import get_timeframe
 from freqtrade.optimize.backtesting import Backtesting
 from freqtrade.strategy.interface import SellType
 from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe,
-                                      _get_frame_time_from_offset)
+                                      _get_frame_time_from_offset, tests_ticker_interval)
 from freqtrade.tests.conftest import patch_exchange
 
 
@@ -147,6 +148,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None:
     """
     default_conf["stoploss"] = data.stop_loss
     default_conf["minimal_roi"] = {"0": data.roi}
+    default_conf['ticker_interval'] = tests_ticker_interval
     mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.0))
     patch_exchange(mocker)
     frame = _build_backtest_dataframe(data.data)
@@ -158,29 +160,21 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None:
     pair = 'UNITTEST/BTC'
     # Dummy data as we mock the analyze functions
     data_processed = {pair: DataFrame()}
+    min_date, max_date = get_timeframe({pair: frame})
     results = backtesting.backtest(
         {
             'stake_amount': default_conf['stake_amount'],
             'processed': data_processed,
             'max_open_trades': 10,
+            'start_date': min_date,
+            'end_date': max_date,
         }
     )
     print(results.T)
 
     assert len(results) == len(data.trades)
     assert round(results["profit_percent"].sum(), 3) == round(data.profit_perc, 3)
-    # if data.sell_r == SellType.STOP_LOSS:
+
-    #     assert log_has("Stop loss hit.", caplog.record_tuples)
-    # else:
-    #     assert not log_has("Stop loss hit.", caplog.record_tuples)
-    # log_test = (f'Force_selling still open trade UNITTEST/BTC with '
-    #             f'{results.iloc[-1].profit_percent} perc - {results.iloc[-1].profit_abs}')
-    # if data.sell_r == SellType.FORCE_SELL:
-    #     assert log_has(log_test,
-    #                    caplog.record_tuples)
-    # else:
-    #     assert not log_has(log_test,
-    #                        caplog.record_tuples)
     for c, trade in enumerate(data.trades):
         res = results.iloc[c]
         assert res.sell_reason == trade.sell_reason

freqtrade/tests/optimize/test_backtesting.py

@@ -13,6 +13,7 @@ from arrow import Arrow
 
 from freqtrade import DependencyException, constants, optimize
 from freqtrade.arguments import Arguments, TimeRange
+from freqtrade.optimize import get_timeframe
 from freqtrade.optimize.backtesting import (Backtesting, setup_configuration,
                                             start)
 from freqtrade.tests.conftest import log_has, patch_exchange
@@ -86,17 +87,21 @@ def load_data_test(what):
 
 def simple_backtest(config, contour, num_results, mocker) -> None:
     patch_exchange(mocker)
+    config['ticker_interval'] = '1m'
     backtesting = Backtesting(config)
 
     data = load_data_test(contour)
     processed = backtesting.strategy.tickerdata_to_dataframe(data)
+    min_date, max_date = get_timeframe(processed)
     assert isinstance(processed, dict)
     results = backtesting.backtest(
         {
             'stake_amount': config['stake_amount'],
             'processed': processed,
             'max_open_trades': 1,
-            'position_stacking': False
+            'position_stacking': False,
+            'start_date': min_date,
+            'end_date': max_date,
         }
     )
     # results :: <class 'pandas.core.frame.DataFrame'>
@@ -123,12 +128,16 @@ def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None):
     data = trim_dictlist(data, -201)
     patch_exchange(mocker)
     backtesting = Backtesting(conf)
+    processed = backtesting.strategy.tickerdata_to_dataframe(data)
+    min_date, max_date = get_timeframe(processed)
     return {
         'stake_amount': conf['stake_amount'],
-        'processed': backtesting.strategy.tickerdata_to_dataframe(data),
+        'processed': processed,
         'max_open_trades': 10,
         'position_stacking': False,
-        'record': record
+        'record': record,
+        'start_date': min_date,
+        'end_date': max_date,
     }
 
 
@@ -449,7 +458,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None:
     )
 
     default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
-    default_conf['ticker_interval'] = "1m"
+    default_conf['ticker_interval'] = '1m'
     default_conf['live'] = False
     default_conf['datadir'] = None
     default_conf['export'] = None
@@ -505,12 +514,15 @@ def test_backtest(default_conf, fee, mocker) -> None:
     data = optimize.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC'])
     data = trim_dictlist(data, -200)
     data_processed = backtesting.strategy.tickerdata_to_dataframe(data)
+    min_date, max_date = get_timeframe(data_processed)
     results = backtesting.backtest(
         {
             'stake_amount': default_conf['stake_amount'],
             'processed': data_processed,
             'max_open_trades': 10,
-            'position_stacking': False
+            'position_stacking': False,
+            'start_date': min_date,
+            'end_date': max_date,
         }
     )
     assert not results.empty
@@ -554,12 +566,16 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None:
     # Run a backtesting for an exiting 5min ticker_interval
     data = optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC'])
     data = trim_dictlist(data, -200)
+    processed = backtesting.strategy.tickerdata_to_dataframe(data)
+    min_date, max_date = get_timeframe(processed)
     results = backtesting.backtest(
         {
             'stake_amount': default_conf['stake_amount'],
-            'processed': backtesting.strategy.tickerdata_to_dataframe(data),
+            'processed': processed,
             'max_open_trades': 1,
-            'position_stacking': False
+            'position_stacking': False,
+            'start_date': min_date,
+            'end_date': max_date,
         }
     )
     assert not results.empty
@@ -583,25 +599,13 @@ def test_processed(default_conf, mocker) -> None:
 def test_backtest_pricecontours(default_conf, fee, mocker) -> None:
     mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
     tests = [['raise', 18], ['lower', 0], ['sine', 19]]
+    # We need to enable sell-signal - otherwise it sells on ROI!!
+    default_conf['experimental'] = {"use_sell_signal": True}
+
     for [contour, numres] in tests:
         simple_backtest(default_conf, contour, numres, mocker)
 
 
-# Test backtest using offline data (testdata directory)
-def test_backtest_ticks(default_conf, fee, mocker):
-    mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
-    patch_exchange(mocker)
-    ticks = [1, 5]
-    fun = Backtesting(default_conf).advise_buy
-    for _ in ticks:
-        backtest_conf = _make_backtest_conf(mocker, conf=default_conf)
-        backtesting = Backtesting(default_conf)
-        backtesting.advise_buy = fun  # Override
-        backtesting.advise_sell = fun  # Override
-        results = backtesting.backtest(backtest_conf)
-        assert not results.empty
-
-
 def test_backtest_clash_buy_sell(mocker, default_conf):
     # Override the default buy trend function in our default_strategy
     def fun(dataframe=None, pair=None):
@@ -636,14 +640,92 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker):
     mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
     mocker.patch('freqtrade.optimize.backtesting.file_dump_json', MagicMock())
     backtest_conf = _make_backtest_conf(mocker, conf=default_conf, pair='UNITTEST/BTC')
+    # We need to enable sell-signal - otherwise it sells on ROI!!
+    default_conf['experimental'] = {"use_sell_signal": True}
+    default_conf['ticker_interval'] = '1m'
     backtesting = Backtesting(default_conf)
     backtesting.advise_buy = _trend_alternate  # Override
     backtesting.advise_sell = _trend_alternate  # Override
     results = backtesting.backtest(backtest_conf)
     backtesting._store_backtest_result("test_.json", results)
-    assert len(results) == 4
+    # 200 candles in backtest data
+    # won't buy on first (shifted by 1)
+    # 100 buys signals
+    assert len(results) == 99
     # One trade was force-closed at the end
-    assert len(results.loc[results.open_at_end]) == 1
+    assert len(results.loc[results.open_at_end]) == 0
+
+
+def test_backtest_multi_pair(default_conf, fee, mocker):
+
+    def evaluate_result_multi(results, freq, max_open_trades):
+            # Find overlapping trades by expanding each trade once per period
+            # and then counting overlaps
+        dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time, freq=freq))
+                 for row in results[['open_time', 'close_time']].iterrows()]
+        deltas = [len(x) for x in dates]
+        dates = pd.Series(pd.concat(dates).values, name='date')
+        df2 = pd.DataFrame(np.repeat(results.values, deltas, axis=0), columns=results.columns)
+
+        df2 = df2.astype(dtype={"open_time": "datetime64", "close_time": "datetime64"})
+        df2 = pd.concat([dates, df2], axis=1)
+        df2 = df2.set_index('date')
+        df_final = df2.resample(freq)[['pair']].count()
+        return df_final[df_final['pair'] > max_open_trades]
+
+    def _trend_alternate_hold(dataframe=None, metadata=None):
+        """
+        Buy every 8th candle - sell every other 8th -2 (hold on to pairs a bit)
+        """
+        multi = 8
+        dataframe['buy'] = np.where(dataframe.index % multi == 0, 1, 0)
+        dataframe['sell'] = np.where((dataframe.index + multi - 2) % multi == 0, 1, 0)
+        if metadata['pair'] in('ETH/BTC', 'LTC/BTC'):
+            dataframe['buy'] = dataframe['buy'].shift(-4)
+            dataframe['sell'] = dataframe['sell'].shift(-4)
+        return dataframe
+
+    mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
+    patch_exchange(mocker)
+    pairs = ['ADA/BTC', 'DASH/BTC', 'ETH/BTC', 'LTC/BTC', 'NXT/BTC']
+    data = optimize.load_data(None, ticker_interval='5m', pairs=pairs)
+    data = trim_dictlist(data, -500)
+    # We need to enable sell-signal - otherwise it sells on ROI!!
+    default_conf['experimental'] = {"use_sell_signal": True}
+    default_conf['ticker_interval'] = '5m'
+
+    backtesting = Backtesting(default_conf)
+    backtesting.advise_buy = _trend_alternate_hold  # Override
+    backtesting.advise_sell = _trend_alternate_hold  # Override
+
+    data_processed = backtesting.strategy.tickerdata_to_dataframe(data)
+    min_date, max_date = get_timeframe(data_processed)
+    backtest_conf = {
+        'stake_amount': default_conf['stake_amount'],
+        'processed': data_processed,
+        'max_open_trades': 3,
+        'position_stacking': False,
+        'start_date': min_date,
+        'end_date': max_date,
+    }
+
+    results = backtesting.backtest(backtest_conf)
+
+    # Make sure we have parallel trades
+    assert len(evaluate_result_multi(results, '5min', 2)) > 0
+    # make sure we don't have trades with more than configured max_open_trades
+    assert len(evaluate_result_multi(results, '5min', 3)) == 0
+
+    backtest_conf = {
+        'stake_amount': default_conf['stake_amount'],
+        'processed': data_processed,
+        'max_open_trades': 1,
+        'position_stacking': False,
+        'start_date': min_date,
+        'end_date': max_date,
+    }
+    results = backtesting.backtest(backtest_conf)
+    assert len(evaluate_result_multi(results, '5min', 1)) == 0
 
 
 def test_backtest_record(default_conf, fee, mocker):

freqtrade/tests/optimize/test_hyperopt.py

@@ -1,11 +1,12 @@
 # pragma pylint: disable=missing-docstring,W0212,C0103
+from datetime import datetime
 import os
 from unittest.mock import MagicMock
 
 import pandas as pd
 import pytest
 
-from freqtrade.optimize.__init__ import load_tickerdata_file
+from freqtrade.optimize import load_tickerdata_file
 from freqtrade.optimize.hyperopt import Hyperopt, start
 from freqtrade.resolvers import StrategyResolver
 from freqtrade.tests.conftest import log_has, patch_exchange
@@ -293,6 +294,10 @@ def test_generate_optimizer(mocker, default_conf) -> None:
         'freqtrade.optimize.hyperopt.Hyperopt.backtest',
         MagicMock(return_value=backtest_result)
     )
+    mocker.patch(
+        'freqtrade.optimize.hyperopt.get_timeframe',
+        MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
+    )
     patch_exchange(mocker)
     mocker.patch('freqtrade.optimize.hyperopt.load', MagicMock())