Merge pull request #513 from gcarq/arrays_for_backtesting

Make backtesting 5x faster

freqtrade/main.py

@@ -296,18 +296,17 @@ def min_roi_reached(trade: Trade, current_rate: float, current_time: datetime) -
     strategy = Strategy()
 
     current_profit = trade.calc_profit_percent(current_rate)
-    if strategy.stoploss is not None and current_profit < float(strategy.stoploss):
+    if strategy.stoploss is not None and current_profit < strategy.stoploss:
         logger.debug('Stop loss hit.')
         return True
 
     # Check if time matches and current rate is above threshold
     time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60
-    for duration_string, threshold in strategy.minimal_roi.items():
+    for duration, threshold in strategy.minimal_roi.items():
-        duration = float(duration_string)
+        if time_diff <= duration:
-        if time_diff > duration and current_profit > threshold:
-            return True
-        if time_diff < duration:
             return False
+        if current_profit > threshold:
+            return True
     return False
 
 

freqtrade/optimize/backtesting.py

@@ -67,30 +67,29 @@ def generate_text_table(
     return tabulate(tabular_data, headers=headers, floatfmt=floatfmt)
 
 
-def get_sell_trade_entry(pair, row, buy_subset, ticker, trade_count_lock, args):
+def get_sell_trade_entry(pair, buy_row, partial_ticker, trade_count_lock, args):
     stake_amount = args['stake_amount']
     max_open_trades = args.get('max_open_trades', 0)
-    trade = Trade(open_rate=row.close,
+    trade = Trade(open_rate=buy_row.close,
-                  open_date=row.Index,
+                  open_date=buy_row.date,
                   stake_amount=stake_amount,
-                  amount=stake_amount / row.open,
+                  amount=stake_amount / buy_row.open,
                   fee=exchange.get_fee()
                   )
 
     # calculate win/lose forwards from buy point
-    sell_subset = ticker[ticker.index > row.Index][['close', 'sell', 'buy']]
+    for sell_row in partial_ticker:
-    for row2 in sell_subset.itertuples(index=True):
         if max_open_trades > 0:
             # Increase trade_count_lock for every iteration
-            trade_count_lock[row2.Index] = trade_count_lock.get(row2.Index, 0) + 1
+            trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1
 
-        buy_signal = row2.buy
+        buy_signal = sell_row.buy
-        if(should_sell(trade, row2.close, row2.Index, buy_signal, row2.sell)):
+        if should_sell(trade, sell_row.close, sell_row.date, buy_signal, sell_row.sell):
-            return row2, (pair,
+            return sell_row, (pair,
-                          trade.calc_profit_percent(rate=row2.close),
+                              trade.calc_profit_percent(rate=sell_row.close),
-                          trade.calc_profit(rate=row2.close),
+                              trade.calc_profit(rate=sell_row.close),
-                          (row2.Index - row.Index).seconds // 60
+                              (sell_row.date - buy_row.date).seconds // 60
-                          ), row2.Index
+                              ), sell_row.date
     return None
 
 
@@ -107,6 +106,7 @@ def backtest(args) -> DataFrame:
         stoploss: use stoploss
     :return: DataFrame
     """
+    headers = ['date', 'buy', 'open', 'close', 'sell']
     processed = args['processed']
     max_open_trades = args.get('max_open_trades', 0)
     realistic = args.get('realistic', True)
@@ -116,29 +116,26 @@ def backtest(args) -> DataFrame:
     trade_count_lock: dict = {}
     exchange._API = Bittrex({'key': '', 'secret': ''})
     for pair, pair_data in processed.items():
-        pair_data['buy'], pair_data['sell'] = 0, 0
+        pair_data['buy'], pair_data['sell'] = 0, 0  # cleanup from previous run
-        ticker = populate_sell_trend(populate_buy_trend(pair_data))
+
-        if 'date' in ticker:
+        ticker_data = populate_sell_trend(populate_buy_trend(pair_data))[headers]
-            ticker.set_index('date', inplace=True)
+        ticker = [x for x in ticker_data.itertuples()]
-        # for each buy point
+
         lock_pair_until = None
-        headers = ['buy', 'open', 'close', 'sell']
+        for index, row in enumerate(ticker):
-        buy_subset = ticker[(ticker.buy == 1) & (ticker.sell == 0)][headers]
+            if row.buy == 0 or row.sell == 1:
-        for row in buy_subset.itertuples(index=True):
+                continue  # skip rows where no buy signal or that would immediately sell off
+
             if realistic:
-                if lock_pair_until is not None and row.Index <= lock_pair_until:
+                if lock_pair_until is not None and row.date <= lock_pair_until:
                     continue
             if max_open_trades > 0:
                 # Check if max_open_trades has already been reached for the given date
-                if not trade_count_lock.get(row.Index, 0) < max_open_trades:
+                if not trade_count_lock.get(row.date, 0) < max_open_trades:
                     continue
+                trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
 
-            if max_open_trades > 0:
+            ret = get_sell_trade_entry(pair, row, ticker[index+1:], trade_count_lock, args)
-                # Increase lock
-                trade_count_lock[row.Index] = trade_count_lock.get(row.Index, 0) + 1
-
-            ret = get_sell_trade_entry(pair, row, buy_subset, ticker,
-                                       trade_count_lock, args)
             if ret:
                 row2, trade_entry, next_date = ret
                 lock_pair_until = next_date
@@ -148,9 +145,9 @@ def backtest(args) -> DataFrame:
                     # record a tuple of pair, current_profit_percent,
                     # entry-date, duration
                     records.append((pair, trade_entry[1],
-                                    row.Index.strftime('%s'),
+                                    row.date.strftime('%s'),
-                                    row2.Index.strftime('%s'),
+                                    row2.date.strftime('%s'),
-                                    row.Index, trade_entry[3]))
+                                    row.date, trade_entry[3]))
     # For now export inside backtest(), maybe change so that backtest()
     # returns a tuple like: (dataframe, records, logs, etc)
     if record and record.find('trades') >= 0:

freqtrade/optimize/hyperopt.py

@@ -225,12 +225,12 @@ def calculate_loss(total_profit: float, trade_count: int, trade_duration: float)
     return trade_loss + profit_loss + duration_loss
 
 
-def generate_roi_table(params) -> Dict[str, float]:
+def generate_roi_table(params) -> Dict[int, float]:
     roi_table = {}
-    roi_table["0"] = params['roi_p1'] + params['roi_p2'] + params['roi_p3']
+    roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3']
-    roi_table[str(params['roi_t3'])] = params['roi_p1'] + params['roi_p2']
+    roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2']
-    roi_table[str(params['roi_t3'] + params['roi_t2'])] = params['roi_p1']
+    roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1']
-    roi_table[str(params['roi_t3'] + params['roi_t2'] + params['roi_t1'])] = 0
+    roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0
 
     return roi_table
 

freqtrade/strategy/strategy.py

@@ -71,11 +71,11 @@ class Strategy(object):
 
         # Minimal ROI designed for the strategy
         self.minimal_roi = OrderedDict(sorted(
-            self.custom_strategy.minimal_roi.items(),
+            {int(key): value for (key, value) in self.custom_strategy.minimal_roi.items()}.items(),
-            key=lambda tuple: float(tuple[0])))  # sort after converting to number
+            key=lambda tuple: tuple[0]))  # sort after converting to number
 
         # Optimal stoploss designed for the strategy
-        self.stoploss = self.custom_strategy.stoploss
+        self.stoploss = float(self.custom_strategy.stoploss)
 
         self.ticker_interval = self.custom_strategy.ticker_interval
 

freqtrade/tests/optimize/test_hyperopt.py

@@ -255,7 +255,7 @@ def test_roi_table_generation():
         'roi_p2': 2,
         'roi_p3': 3,
     }
-    assert generate_roi_table(params) == {'0': 6, '15': 3, '25': 1, '30': 0}
+    assert generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0}
 
 
 # test log_trials_result

freqtrade/tests/strategy/test_strategy.py

@@ -57,7 +57,7 @@ def test_strategy(result):
     strategy.init({'strategy': 'default_strategy'})
 
     assert hasattr(strategy.custom_strategy, 'minimal_roi')
-    assert strategy.minimal_roi['0'] == 0.04
+    assert strategy.minimal_roi[0] == 0.04
 
     assert hasattr(strategy.custom_strategy, 'stoploss')
     assert strategy.stoploss == -0.10
@@ -86,7 +86,7 @@ def test_strategy_override_minimal_roi(caplog):
     strategy.init(config)
 
     assert hasattr(strategy.custom_strategy, 'minimal_roi')
-    assert strategy.minimal_roi['0'] == 0.5
+    assert strategy.minimal_roi[0] == 0.5
     assert ('freqtrade.strategy.strategy',
             logging.INFO,
             'Override strategy \'minimal_roi\' with value in config file.'
@@ -142,8 +142,8 @@ def test_strategy_singleton():
     strategy1.init({'strategy': 'default_strategy'})
 
     assert hasattr(strategy1.custom_strategy, 'minimal_roi')
-    assert strategy1.minimal_roi['0'] == 0.04
+    assert strategy1.minimal_roi[0] == 0.04
 
     strategy2 = Strategy()
     assert hasattr(strategy2.custom_strategy, 'minimal_roi')
-    assert strategy2.minimal_roi['0'] == 0.04
+    assert strategy2.minimal_roi[0] == 0.04