Merge branch 'freqtrade:develop' into develop

freqtrade/data/dataprovider.py

@@ -149,6 +149,8 @@ class DataProvider:
         Clear pair dataframe cache.
         """
         self.__cached_pairs = {}
+        self.__cached_pairs_backtesting = {}
+        self.__slice_index = 0
 
     # Exchange functions
 

freqtrade/optimize/backtesting.py

@@ -85,18 +85,7 @@ class Backtesting:
                                        "configuration or as cli argument `--timeframe 5m`")
         self.timeframe = str(self.config.get('timeframe'))
         self.timeframe_min = timeframe_to_minutes(self.timeframe)
-        # Load detail timeframe if specified
+        self.init_backtest_detail()
-        self.timeframe_detail = str(self.config.get('timeframe_detail', ''))
-        if self.timeframe_detail:
-            self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail)
-            if self.timeframe_min <= self.timeframe_detail_min:
-                raise OperationalException(
-                    "Detail timeframe must be smaller than strategy timeframe.")
-
-        else:
-            self.timeframe_detail_min = 0
-        self.detail_data: Dict[str, DataFrame] = {}
-
         self.pairlists = PairListManager(self.exchange, self.config)
         if 'VolumePairList' in self.pairlists.name_list:
             raise OperationalException("VolumePairList not allowed for backtesting.")
@@ -119,14 +108,6 @@ class Backtesting:
         else:
             self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0])
 
-        Trade.use_db = False
-        Trade.reset_trades()
-        PairLocks.timeframe = self.config['timeframe']
-        PairLocks.use_db = False
-        PairLocks.reset_locks()
-
-        self.wallets = Wallets(self.config, self.exchange, log=False)
-
         self.timerange = TimeRange.parse_timerange(
             None if self.config.get('timerange') is None else str(self.config.get('timerange')))
 
@@ -135,9 +116,7 @@ class Backtesting:
         # Add maximum startup candle count to configuration for informative pairs support
         self.config['startup_candle_count'] = self.required_startup
         self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
-
+        self.init_backtest()
-        self.progress = BTProgress()
-        self.abort = False
 
     def __del__(self):
         self.cleanup()
@@ -147,6 +126,28 @@ class Backtesting:
         PairLocks.use_db = True
         Trade.use_db = True
 
+    def init_backtest_detail(self):
+        # Load detail timeframe if specified
+        self.timeframe_detail = str(self.config.get('timeframe_detail', ''))
+        if self.timeframe_detail:
+            self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail)
+            if self.timeframe_min <= self.timeframe_detail_min:
+                raise OperationalException(
+                    "Detail timeframe must be smaller than strategy timeframe.")
+
+        else:
+            self.timeframe_detail_min = 0
+        self.detail_data: Dict[str, DataFrame] = {}
+
+    def init_backtest(self):
+
+        self.prepare_backtest(False)
+
+        self.wallets = Wallets(self.config, self.exchange, log=False)
+
+        self.progress = BTProgress()
+        self.abort = False
+
     def _set_strategy(self, strategy: IStrategy):
         """
         Load strategy into backtesting
@@ -226,7 +227,8 @@ class Backtesting:
         Trade.reset_trades()
         self.rejected_trades = 0
         self.dataprovider.clear_cache()
-        self._load_protections(self.strategy)
+        if enable_protections:
+            self._load_protections(self.strategy)
 
     def check_abort(self):
         """

freqtrade/rpc/api_server/api_backtest.py

@@ -47,33 +47,34 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
                 not ApiServer._bt
                 or lastconfig.get('timeframe') != strat.timeframe
                 or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail')
-                or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0)
                 or lastconfig.get('timerange') != btconfig['timerange']
             ):
                 from freqtrade.optimize.backtesting import Backtesting
                 ApiServer._bt = Backtesting(btconfig)
                 if ApiServer._bt.timeframe_detail:
                     ApiServer._bt.load_bt_data_detail()
-
+            else:
+                ApiServer._bt.config = btconfig
+                ApiServer._bt.init_backtest()
             # Only reload data if timeframe changed.
             if (
                 not ApiServer._bt_data
                 or not ApiServer._bt_timerange
-                or lastconfig.get('stake_amount') != btconfig.get('stake_amount')
-                or lastconfig.get('enable_protections') != btconfig.get('enable_protections')
-                or lastconfig.get('protections') != btconfig.get('protections', [])
                 or lastconfig.get('timeframe') != strat.timeframe
             ):
-                lastconfig['timerange'] = btconfig['timerange']
-                lastconfig['protections'] = btconfig.get('protections', [])
-                lastconfig['enable_protections'] = btconfig.get('enable_protections')
-                lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet')
-                lastconfig['timeframe'] = strat.timeframe
                 ApiServer._bt_data, ApiServer._bt_timerange = ApiServer._bt.load_bt_data()
 
+                lastconfig['timerange'] = btconfig['timerange']
+                lastconfig['timeframe'] = strat.timeframe
+
+            lastconfig['protections'] = btconfig.get('protections', [])
+            lastconfig['enable_protections'] = btconfig.get('enable_protections')
+            lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet')
+
             ApiServer._bt.abort = False
             min_date, max_date = ApiServer._bt.backtest_one_strategy(
                 strat, ApiServer._bt_data, ApiServer._bt_timerange)
+
             ApiServer._bt.results = generate_backtest_stats(
                 ApiServer._bt_data, ApiServer._bt.all_results,
                 min_date=min_date, max_date=max_date)

tests/conftest.py

@@ -1685,14 +1685,6 @@ def trades_for_order2():
              'fee': {'cost': 0.004, 'currency': 'LTC'}}]
 
 
-@pytest.fixture(scope="function")
-def trades_for_order3(trades_for_order2):
-    # Different fee currencies for each trade
-    trades_for_order = deepcopy(trades_for_order2)
-    trades_for_order[0]['fee'] = {'cost': 0.02, 'currency': 'BNB'}
-    return trades_for_order
-
-
 @pytest.fixture
 def buy_order_fee():
     return {

tests/test_freqtradebot.py

@@ -3568,8 +3568,33 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f
                    caplog)
 
 
-def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fee, mocker):
+@pytest.mark.parametrize(
-    trades_for_order[0]['fee']['currency'] = 'ETH'
+    'fee_par,fee_reduction_amount,use_ticker_rate,expected_log', [
+        # basic, amount does not change
+        ({'cost': 0.008, 'currency': 'ETH'}, 0, False, None),
+        # no currency in fee
+        ({'cost': 0.004, 'currency': None}, 0, True, None),
+        # BNB no rate
+        ({'cost': 0.00094518, 'currency': 'BNB'}, 0, True, (
+            'Fee for Trade Trade(id=None, pair=LTC/ETH, amount=8.00000000, open_rate=0.24544100,'
+            ' open_since=closed) [buy]: 0.00094518 BNB - rate: None'
+        )),
+        # from order
+        ({'cost': 0.004, 'currency': 'LTC'}, 0.004, False, (
+            'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
+            'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).'
+        )),
+        # invalid, no currency in from fee dict
+        ({'cost': 0.008, 'currency': None}, 0, True, None),
+    ])
+def test_get_real_amount(
+    default_conf, trades_for_order, buy_order_fee, fee, mocker, caplog,
+    fee_par, fee_reduction_amount, use_ticker_rate, expected_log
+):
+
+    buy_order = deepcopy(buy_order_fee)
+    buy_order['fee'] = fee_par
+    trades_for_order[0]['fee'] = fee_par
 
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
     amount = sum(x['amount'] for x in trades_for_order)
@@ -3584,19 +3609,38 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fe
     )
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
-    # Amount does not change
+    if not use_ticker_rate:
-    assert freqtrade.get_real_amount(trade, buy_order_fee) == amount
+        mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', side_effect=ExchangeError)
+
+    caplog.clear()
+    assert freqtrade.get_real_amount(trade, buy_order) == amount - fee_reduction_amount
+
+    if expected_log:
+        assert log_has(expected_log, caplog)
 
 
-def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_order_fee,
+@pytest.mark.parametrize(
-                                            fee, mocker):
+    'fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount', [
+        # basic, amount is reduced by fee
+        (None, None, 0.001, 0.001, 7.992),
+        # different fee currency on both trades, fee is average of both trade's fee
+        (0.02, 'BNB', 0.0005, 0.001518575, 7.996),
+    ])
+def test_get_real_amount_multi(
+    default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker, markets,
+    fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount,
+):
 
-    limit_buy_order = deepcopy(buy_order_fee)
+    trades_for_order = deepcopy(trades_for_order2)
-    limit_buy_order['fee'] = {'cost': 0.004, 'currency': None}
+    if fee_cost:
-    trades_for_order[0]['fee']['currency'] = None
+        trades_for_order[0]['fee']['cost'] = fee_cost
+    if fee_currency:
+        trades_for_order[0]['fee']['currency'] = fee_currency
 
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
-    amount = sum(x['amount'] for x in trades_for_order)
+    amount = float(sum(x['amount'] for x in trades_for_order))
+    default_conf['stake_currency'] = "ETH"
+
     trade = Trade(
         pair='LTC/ETH',
         amount=amount,
@@ -3606,76 +3650,7 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
-    # Amount does not change
-    assert freqtrade.get_real_amount(trade, limit_buy_order) == amount
-
-
-def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, fee, mocker):
-    trades_for_order[0]['fee']['currency'] = 'BNB'
-    trades_for_order[0]['fee']['cost'] = 0.00094518
-
-    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
-    amount = sum(x['amount'] for x in trades_for_order)
-    trade = Trade(
-        pair='LTC/ETH',
-        amount=amount,
-        exchange='binance',
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        open_rate=0.245441,
-        open_order_id="123456"
-    )
-    freqtrade = get_patched_freqtradebot(mocker, default_conf)
-
-    # Amount does not change
-    assert freqtrade.get_real_amount(trade, buy_order_fee) == amount
-
-
-def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker):
-    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order2)
-    amount = float(sum(x['amount'] for x in trades_for_order2))
-    trade = Trade(
-        pair='LTC/ETH',
-        amount=amount,
-        exchange='binance',
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        open_rate=0.245441,
-        open_order_id="123456"
-    )
-    freqtrade = get_patched_freqtradebot(mocker, default_conf)
-
-    # Amount is reduced by "fee"
-    assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001)
-    assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
-                   'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).',
-                   caplog)
-
-    assert trade.fee_open == 0.001
-    assert trade.fee_close == 0.001
-    assert trade.fee_open_cost is not None
-    assert trade.fee_open_currency is not None
-    assert trade.fee_close_cost is None
-    assert trade.fee_close_currency is None
-
-
-def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, caplog, fee,
-                                mocker, markets):
-    # Different fee currency on both trades
-    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order3)
-    amount = float(sum(x['amount'] for x in trades_for_order3))
-    default_conf['stake_currency'] = 'ETH'
-    trade = Trade(
-        pair='LTC/ETH',
-        amount=amount,
-        exchange='binance',
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        open_rate=0.245441,
-        open_order_id="123456"
-    )
     # Fake markets entry to enable fee parsing
     markets['BNB/ETH'] = markets['ETH/BTC']
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
@@ -3684,46 +3659,24 @@ def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee,
                  return_value={'ask': 0.19, 'last': 0.2})
 
     # Amount is reduced by "fee"
-    assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.0005)
+    expected_amount = amount - (amount * fee_reduction_amount)
-    assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
+    assert freqtrade.get_real_amount(trade, buy_order_fee) == expected_amount
-                   'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).',
+    assert log_has(
-                   caplog)
+        (
-    # Overall fee is average of both trade's fee
+            'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
-    assert trade.fee_open == 0.001518575
+            f'open_rate=0.24544100, open_since=closed) (from 8.0 to {expected_log_amount}).'
+        ),
+        caplog
+    )
+
+    assert trade.fee_open == expected_fee
+    assert trade.fee_close == expected_fee
     assert trade.fee_open_cost is not None
     assert trade.fee_open_currency is not None
     assert trade.fee_close_cost is None
     assert trade.fee_close_currency is None
 
 
-def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, fee,
-                                   caplog, mocker):
-    limit_buy_order = deepcopy(buy_order_fee)
-    limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'}
-
-    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order',
-                 return_value=[trades_for_order])
-    amount = float(sum(x['amount'] for x in trades_for_order))
-    trade = Trade(
-        pair='LTC/ETH',
-        amount=amount,
-        exchange='binance',
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        open_rate=0.245441,
-        open_order_id="123456"
-    )
-    freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    # Ticker rate cannot be found for this to work.
-    mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', side_effect=ExchangeError)
-
-    # Amount is reduced by "fee"
-    assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004
-    assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
-                   'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).',
-                   caplog)
-
-
 def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, fee, mocker):
     limit_buy_order = deepcopy(buy_order_fee)
     limit_buy_order['fee'] = {'cost': 0.004}
@@ -3791,27 +3744,6 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b
                    abs_tol=MATH_CLOSE_PREC,)
 
 
-def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, fee, mocker):
-    # Remove "Currency" from fee dict
-    trades_for_order[0]['fee'] = {'cost': 0.008}
-
-    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
-    amount = sum(x['amount'] for x in trades_for_order)
-    trade = Trade(
-        pair='LTC/ETH',
-        amount=amount,
-        exchange='binance',
-        open_rate=0.245441,
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-
-        open_order_id="123456"
-    )
-    freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    # Amount does not change
-    assert freqtrade.get_real_amount(trade, buy_order_fee) == amount
-
-
 def test_get_real_amount_open_trade(default_conf, fee, mocker):
     amount = 12345
     trade = Trade(
@@ -3862,10 +3794,14 @@ def test_apply_fee_conditional(default_conf, fee, caplog, mocker,
     assert walletmock.call_count == 1
 
 
+@pytest.mark.parametrize("delta, is_high_delta", [
+    (0.1, False),
+    (100, True),
+])
 def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open, limit_buy_order,
-                                    fee, mocker, order_book_l2):
+                                    fee, mocker, order_book_l2, delta, is_high_delta):
     default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True
-    default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1
+    default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = delta
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2)
@@ -3883,42 +3819,22 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open,
     freqtrade.enter_positions()
 
     trade = Trade.query.first()
-    assert trade is not None
+    if is_high_delta:
-    assert trade.stake_amount == 0.001
+        assert trade is None
-    assert trade.is_open
+    else:
-    assert trade.open_date is not None
+        assert trade is not None
-    assert trade.exchange == 'binance'
+        assert trade.stake_amount == 0.001
+        assert trade.is_open
+        assert trade.open_date is not None
+        assert trade.exchange == 'binance'
 
-    assert len(Trade.query.all()) == 1
+        assert len(Trade.query.all()) == 1
 
-    # Simulate fulfilled LIMIT_BUY order for trade
+        # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
+        trade.update(limit_buy_order)
 
-    assert trade.open_rate == 0.00001099
+        assert trade.open_rate == 0.00001099
-    assert whitelist == default_conf['exchange']['pair_whitelist']
+        assert whitelist == default_conf['exchange']['pair_whitelist']
-
-
-def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_order,
-                                               fee, mocker, order_book_l2):
-    default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True
-    # delta is 100 which is impossible to reach. hence check_depth_of_market will return false
-    default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
-    mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2)
-    mocker.patch.multiple(
-        'freqtrade.exchange.Exchange',
-        fetch_ticker=ticker,
-        create_order=MagicMock(return_value={'id': limit_buy_order['id']}),
-        get_fee=fee,
-    )
-    # Save state of current whitelist
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
-    freqtrade.enter_positions()
-
-    trade = Trade.query.first()
-    assert trade is None
 
 
 @pytest.mark.parametrize('exception_thrown,ask,last,order_book_top,order_book', [