Merge pull request #1683 from gianlup/fix_bt_partial_data

Fix backtest problem with partial data

freqtrade/optimize/backtesting.py

@@ -210,6 +210,32 @@ class Backtesting(object):
             logger.info('Dumping backtest results to %s', recordfilename)
             file_dump_json(recordfilename, records)
 
+    def _get_ticker_list(self, processed) -> Dict[str, DataFrame]:
+        """
+        Helper function to convert a processed tickerlist into a list for performance reasons.
+
+        Used by backtest() - so keep this optimized for performance.
+        """
+        headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
+        ticker: Dict = {}
+        # Create ticker dict
+        for pair, pair_data in processed.items():
+            pair_data['buy'], pair_data['sell'] = 0, 0  # cleanup from previous run
+
+            ticker_data = self.advise_sell(
+                self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
+
+            # to avoid using data from future, we buy/sell with signal from previous candle
+            ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1)
+            ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1)
+
+            ticker_data.drop(ticker_data.head(1).index, inplace=True)
+
+            # Convert from Pandas to list for performance reasons
+            # (Looping Pandas is slow.)
+            ticker[pair] = [x for x in ticker_data.itertuples()]
+        return ticker
+
     def _get_sell_trade_entry(
             self, pair: str, buy_row: DataFrame,
             partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[BacktestResult]:
@@ -304,7 +330,6 @@ class Backtesting(object):
             position_stacking: do we allow position stacking? (default: False)
         :return: DataFrame
         """
-        headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
         processed = args['processed']
         max_open_trades = args.get('max_open_trades', 0)
         position_stacking = args.get('position_stacking', False)
@@ -312,54 +337,50 @@ class Backtesting(object):
         end_date = args['end_date']
         trades = []
         trade_count_lock: Dict = {}
-        ticker: Dict = {}
-        pairs = []
-        # Create ticker dict
-        for pair, pair_data in processed.items():
-            pair_data['buy'], pair_data['sell'] = 0, 0  # cleanup from previous run
 
-            ticker_data = self.advise_sell(
+        # Dict of ticker-lists for performance (looping lists is a lot faster than dataframes)
-                self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
+        ticker: Dict = self._get_ticker_list(processed)
-
-            # to avoid using data from future, we buy/sell with signal from previous candle
-            ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1)
-            ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1)
-
-            ticker_data.drop(ticker_data.head(1).index, inplace=True)
-
-            # Convert from Pandas to list for performance reasons
-            # (Looping Pandas is slow.)
-            ticker[pair] = [x for x in ticker_data.itertuples()]
-            pairs.append(pair)
 
         lock_pair_until: Dict = {}
+        # Indexes per pair, so some pairs are allowed to have a missing start.
+        indexes: Dict = {}
         tmp = start_date + timedelta(minutes=self.ticker_interval_mins)
-        index = 0
+
-        # Loop timerange and test per pair
+        # Loop timerange and get candle for each pair at that point in time
         while tmp < end_date:
-            # print(f"time: {tmp}")
+
             for i, pair in enumerate(ticker):
+                if pair not in indexes:
+                    indexes[pair] = 0
+
                 try:
-                    row = ticker[pair][index]
+                    row = ticker[pair][indexes[pair]]
                 except IndexError:
-                    # missing Data for one pair ...
+                    # missing Data for one pair at the end.
                     # Warnings for this are shown by `validate_backtest_data`
                     continue
 
+                # Waits until the time-counter reaches the start of the data for this pair.
+                if row.date > tmp.datetime:
+                    continue
+
+                indexes[pair] += 1
+
                 if row.buy == 0 or row.sell == 1:
                     continue  # skip rows where no buy signal or that would immediately sell off
 
-                if not position_stacking:
+                if (not position_stacking and pair in lock_pair_until
-                    if pair in lock_pair_until and row.date <= lock_pair_until[pair]:
+                        and row.date <= lock_pair_until[pair]):
-                        continue
+                    # without positionstacking, we can only have one open trade per pair.
+                    continue
+
                 if max_open_trades > 0:
                     # Check if max_open_trades has already been reached for the given date
                     if not trade_count_lock.get(row.date, 0) < max_open_trades:
                         continue
-
                     trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
 
-                trade_entry = self._get_sell_trade_entry(pair, row, ticker[pair][index + 1:],
+                trade_entry = self._get_sell_trade_entry(pair, row, ticker[pair][indexes[pair]:],
                                                          trade_count_lock, args)
 
                 if trade_entry:
@@ -367,11 +388,10 @@ class Backtesting(object):
                     trades.append(trade_entry)
                 else:
                     # Set lock_pair_until to end of testing period if trade could not be closed
-                    # This happens only if the buy-signal was with the last candle
+                    lock_pair_until[pair] = end_date.datetime
-                    lock_pair_until[pair] = end_date
 
+            # Move time one configured time_interval ahead.
             tmp += timedelta(minutes=self.ticker_interval_mins)
-            index += 1
         return DataFrame.from_records(trades, columns=BacktestResult._fields)
 
     def start(self) -> None:

freqtrade/tests/edge/test_edge.py

@@ -122,8 +122,8 @@ def test_edge_results(edge_conf, mocker, caplog, data) -> None:
     for c, trade in enumerate(data.trades):
         res = results.iloc[c]
         assert res.exit_type == trade.sell_reason
-        assert res.open_time == _get_frame_time_from_offset(trade.open_tick)
+        assert arrow.get(res.open_time) == _get_frame_time_from_offset(trade.open_tick)
-        assert res.close_time == _get_frame_time_from_offset(trade.close_tick)
+        assert arrow.get(res.close_time) == _get_frame_time_from_offset(trade.close_tick)
 
 
 def test_adjust(mocker, edge_conf):

freqtrade/tests/optimize/__init__.py

@@ -33,7 +33,7 @@ class BTContainer(NamedTuple):
 
 def _get_frame_time_from_offset(offset):
     return ticker_start_time.shift(minutes=(offset * TICKER_INTERVAL_MINUTES[tests_ticker_interval])
-                                   ).datetime.replace(tzinfo=None)
+                                   ).datetime
 
 
 def _build_backtest_dataframe(ticker_with_signals):

freqtrade/tests/optimize/test_backtesting.py

@@ -685,25 +685,32 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker):
     assert len(results.loc[results.open_at_end]) == 0
 
 
-def test_backtest_multi_pair(default_conf, fee, mocker):
+@pytest.mark.parametrize("pair", ['ADA/BTC', 'LTC/BTC'])
+@pytest.mark.parametrize("tres", [0, 20, 30])
+def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair):
 
     def _trend_alternate_hold(dataframe=None, metadata=None):
         """
-        Buy every 8th candle - sell every other 8th -2 (hold on to pairs a bit)
+        Buy every xth candle - sell every other xth -2 (hold on to pairs a bit)
         """
-        multi = 8
+        if metadata['pair'] in('ETH/BTC', 'LTC/BTC'):
+            multi = 20
+        else:
+            multi = 18
         dataframe['buy'] = np.where(dataframe.index % multi == 0, 1, 0)
         dataframe['sell'] = np.where((dataframe.index + multi - 2) % multi == 0, 1, 0)
-        if metadata['pair'] in('ETH/BTC', 'LTC/BTC'):
-            dataframe['buy'] = dataframe['buy'].shift(-4)
-            dataframe['sell'] = dataframe['sell'].shift(-4)
         return dataframe
 
     mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
     patch_exchange(mocker)
+
     pairs = ['ADA/BTC', 'DASH/BTC', 'ETH/BTC', 'LTC/BTC', 'NXT/BTC']
     data = history.load_data(datadir=None, ticker_interval='5m', pairs=pairs)
+    # Only use 500 lines to increase performance
     data = trim_dictlist(data, -500)
+
+    # Remove data for one pair from the beginning of the data
+    data[pair] = data[pair][tres:]
     # We need to enable sell-signal - otherwise it sells on ROI!!
     default_conf['experimental'] = {"use_sell_signal": True}
     default_conf['ticker_interval'] = '5m'