Merge pull request #4069 from freqtrade/refactor_ohlcv_download

Refactor pairlist ohlcv download to use async

freqtrade/commands/arguments.py

@@ -44,7 +44,8 @@ ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"]
 ARGS_LIST_PAIRS = ["exchange", "print_list", "list_pairs_print_json", "print_one_column",
                    "print_csv", "base_currencies", "quote_currencies", "list_pairs_all"]
 
-ARGS_TEST_PAIRLIST = ["config", "quote_currencies", "print_one_column", "list_pairs_print_json"]
+ARGS_TEST_PAIRLIST = ["verbosity", "config", "quote_currencies", "print_one_column",
+                      "list_pairs_print_json"]
 
 ARGS_CREATE_USERDIR = ["user_data_dir", "reset"]
 

freqtrade/exchange/exchange.py

@@ -733,13 +733,17 @@ class Exchange:
         logger.info("Downloaded data for %s with length %s.", pair, len(data))
         return data
 
-    def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes) -> List[Tuple[str, List]]:
+    def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *,
+                             since_ms: Optional[int] = None, cache: bool = True
+                             ) -> Dict[Tuple[str, str], DataFrame]:
         """
         Refresh in-memory OHLCV asynchronously and set `_klines` with the result
         Loops asynchronously over pair_list and downloads all pairs async (semi-parallel).
         Only used in the dataprovider.refresh() method.
         :param pair_list: List of 2 element tuples containing pair, interval to refresh
-        :return: TODO: return value is only used in the tests, get rid of it
+        :param since_ms: time since when to download, in milliseconds
+        :param cache: Assign result to _klines. Usefull for one-off downloads like for pairlists
+        :return: Dict of [{(pair, timeframe): Dataframe}]
         """
         logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list))
 
@@ -749,7 +753,8 @@ class Exchange:
         for pair, timeframe in set(pair_list):
             if (not ((pair, timeframe) in self._klines)
                     or self._now_is_time_to_refresh(pair, timeframe)):
-                input_coroutines.append(self._async_get_candle_history(pair, timeframe))
+                input_coroutines.append(self._async_get_candle_history(pair, timeframe,
+                                                                       since_ms=since_ms))
             else:
                 logger.debug(
                     "Using cached candle (OHLCV) data for pair %s, timeframe %s ...",
@@ -759,6 +764,7 @@ class Exchange:
         results = asyncio.get_event_loop().run_until_complete(
             asyncio.gather(*input_coroutines, return_exceptions=True))
 
+        results_df = {}
         # handle caching
         for res in results:
             if isinstance(res, Exception):
@@ -770,11 +776,13 @@ class Exchange:
             if ticks:
                 self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000
             # keeping parsed dataframe in cache
-            self._klines[(pair, timeframe)] = ohlcv_to_dataframe(
+            ohlcv_df = ohlcv_to_dataframe(
-                ticks, timeframe, pair=pair, fill_missing=True,
+                    ticks, timeframe, pair=pair, fill_missing=True,
-                drop_incomplete=self._ohlcv_partial_candle)
+                    drop_incomplete=self._ohlcv_partial_candle)
-
+            results_df[(pair, timeframe)] = ohlcv_df
-        return results
+            if cache:
+                self._klines[(pair, timeframe)] = ohlcv_df
+        return results_df
 
     def _now_is_time_to_refresh(self, pair: str, timeframe: str) -> bool:
         # Timeframe in seconds

freqtrade/pairlist/AgeFilter.py

@@ -2,7 +2,7 @@
 Minimum age (days listed) pair list filter
 """
 import logging
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 import arrow
 
@@ -49,35 +49,32 @@ class AgeFilter(IPairList):
         return (f"{self.name} - Filtering pairs with age less than "
                 f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.")
 
-    def _validate_pair(self, ticker: Dict) -> bool:
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
-        Validate age for the ticker
+        :param pairlist: pairlist to filter or sort
-        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
-        :return: True if the pair can stay, False if it should be removed
+        :return: new allowlist
         """
-
+        needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked]
-        # Check symbol in cache
+        if not needed_pairs:
-        if ticker['symbol'] in self._symbolsChecked:
+            return pairlist
-            return True
 
         since_ms = int(arrow.utcnow()
                        .floor('day')
-                       .shift(days=-self._min_days_listed)
+                       .shift(days=-self._min_days_listed - 1)
                        .float_timestamp) * 1000
+        candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
+        pairlist_new = []
+        if self._enabled:
+            for p, _ in needed_pairs:
 
-        daily_candles = self._exchange.get_historic_ohlcv(pair=ticker['symbol'],
+                age = len(candles[(p, '1d')]) if (p, '1d') in candles else 0
-                                                          timeframe='1d',
+                if age > self._min_days_listed:
-                                                          since_ms=since_ms)
+                    pairlist_new.append(p)
-
+                    self._symbolsChecked[p] = int(arrow.utcnow().float_timestamp) * 1000
-        if daily_candles is not None:
+                else:
-            if len(daily_candles) > self._min_days_listed:
+                    self.log_once(f"Removed {p} from whitelist, because age "
-                # We have fetched at least the minimum required number of daily candles
+                                  f"{age} is less than {self._min_days_listed} "
-                # Add to cache, store the time we last checked this symbol
+                                  f"{plural(self._min_days_listed, 'day')}", logger.info)
-                self._symbolsChecked[ticker['symbol']] = int(arrow.utcnow().float_timestamp) * 1000
+        logger.info(f"Validated {len(pairlist_new)} pairs.")
-                return True
+        return pairlist_new
-            else:
-                self.log_once(f"Removed {ticker['symbol']} from whitelist, because age "
-                              f"{len(daily_candles)} is less than {self._min_days_listed} "
-                              f"{plural(self._min_days_listed, 'day')}", logger.info)
-                return False
-        return False

freqtrade/pairlist/IPairList.py

@@ -60,13 +60,14 @@ class IPairList(LoggingMixin, ABC):
         -> Please overwrite in subclasses
         """
 
-    def _validate_pair(self, ticker) -> bool:
+    def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool:
         """
         Check one pair against Pairlist Handler's specific conditions.
 
         Either implement it in the Pairlist Handler or override the generic
         filter_pairlist() method.
 
+        :param pair: Pair that's currently validated
         :param ticker: ticker dict as returned from ccxt.load_markets()
         :return: True if the pair can stay, false if it should be removed
         """
@@ -109,7 +110,7 @@ class IPairList(LoggingMixin, ABC):
             # Copy list since we're modifying this list
             for p in deepcopy(pairlist):
                 # Filter out assets
-                if not self._validate_pair(tickers[p]):
+                if not self._validate_pair(p, tickers[p] if p in tickers else {}):
                     pairlist.remove(p)
 
         return pairlist

freqtrade/pairlist/PrecisionFilter.py

@@ -43,19 +43,20 @@ class PrecisionFilter(IPairList):
         """
         return f"{self.name} - Filtering untradable pairs."
 
-    def _validate_pair(self, ticker: dict) -> bool:
+    def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool:
         """
         Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very
         low value pairs.
+        :param pair: Pair that's currently validated
         :param ticker: ticker dict as returned from ccxt.load_markets()
-        :return: True if the pair can stay, False if it should be removed
+        :return: True if the pair can stay, false if it should be removed
         """
         stop_price = ticker['ask'] * self._stoploss
 
         # Adjust stop-prices to precision
-        sp = self._exchange.price_to_precision(ticker["symbol"], stop_price)
+        sp = self._exchange.price_to_precision(pair, stop_price)
 
-        stop_gap_price = self._exchange.price_to_precision(ticker["symbol"], stop_price * 0.99)
+        stop_gap_price = self._exchange.price_to_precision(pair, stop_price * 0.99)
         logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}")
 
         if sp <= stop_gap_price:

freqtrade/pairlist/PriceFilter.py

@@ -57,31 +57,32 @@ class PriceFilter(IPairList):
 
         return f"{self.name} - No price filters configured."
 
-    def _validate_pair(self, ticker) -> bool:
+    def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool:
         """
         Check if if one price-step (pip) is > than a certain barrier.
+        :param pair: Pair that's currently validated
         :param ticker: ticker dict as returned from ccxt.load_markets()
         :return: True if the pair can stay, false if it should be removed
         """
         if ticker['last'] is None or ticker['last'] == 0:
-            self.log_once(f"Removed {ticker['symbol']} from whitelist, because "
+            self.log_once(f"Removed {pair} from whitelist, because "
                           "ticker['last'] is empty (Usually no trade in the last 24h).",
                           logger.info)
             return False
 
         # Perform low_price_ratio check.
         if self._low_price_ratio != 0:
-            compare = self._exchange.price_get_one_pip(ticker['symbol'], ticker['last'])
+            compare = self._exchange.price_get_one_pip(pair, ticker['last'])
             changeperc = compare / ticker['last']
             if changeperc > self._low_price_ratio:
-                self.log_once(f"Removed {ticker['symbol']} from whitelist, "
+                self.log_once(f"Removed {pair} from whitelist, "
                               f"because 1 unit is {changeperc * 100:.3f}%", logger.info)
                 return False
 
         # Perform min_price check.
         if self._min_price != 0:
             if ticker['last'] < self._min_price:
-                self.log_once(f"Removed {ticker['symbol']} from whitelist, "
+                self.log_once(f"Removed {pair} from whitelist, "
                               f"because last price < {self._min_price:.8f}", logger.info)
                 return False
 

freqtrade/pairlist/SpreadFilter.py

@@ -36,16 +36,17 @@ class SpreadFilter(IPairList):
         return (f"{self.name} - Filtering pairs with ask/bid diff above "
                 f"{self._max_spread_ratio * 100}%.")
 
-    def _validate_pair(self, ticker: dict) -> bool:
+    def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool:
         """
         Validate spread for the ticker
+        :param pair: Pair that's currently validated
         :param ticker: ticker dict as returned from ccxt.load_markets()
-        :return: True if the pair can stay, False if it should be removed
+        :return: True if the pair can stay, false if it should be removed
         """
         if 'bid' in ticker and 'ask' in ticker:
             spread = 1 - ticker['bid'] / ticker['ask']
             if spread > self._max_spread_ratio:
-                self.log_once(f"Removed {ticker['symbol']} from whitelist, because spread "
+                self.log_once(f"Removed {pair} from whitelist, because spread "
                               f"{spread * 100:.3f}% > {self._max_spread_ratio * 100}%",
                               logger.info)
                 return False

freqtrade/pairlist/pairlistmanager.py

@@ -3,7 +3,7 @@ PairList manager class
 """
 import logging
 from copy import deepcopy
-from typing import Dict, List
+from typing import Any, Dict, List
 
 from cachetools import TTLCache, cached
 
@@ -97,7 +97,7 @@ class PairListManager():
 
         self._whitelist = pairlist
 
-    def _prepare_whitelist(self, pairlist: List[str], tickers) -> List[str]:
+    def _prepare_whitelist(self, pairlist: List[str], tickers: Dict[str, Any]) -> List[str]:
         """
         Prepare sanitized pairlist for Pairlist Handlers that use tickers data - remove
         pairs that do not have ticker available

freqtrade/pairlist/rangestabilityfilter.py

@@ -2,10 +2,12 @@
 Rate of change pairlist filter
 """
 import logging
-from typing import Any, Dict
+from copy import deepcopy
+from typing import Any, Dict, List, Optional
 
 import arrow
 from cachetools.ttl import TTLCache
+from pandas import DataFrame
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import plural
@@ -42,7 +44,7 @@ class RangeStabilityFilter(IPairList):
         If no Pairlist requires tickers, an empty List is passed
         as tickers argument to filter_pairlist
         """
-        return True
+        return False
 
     def short_desc(self) -> str:
         """
@@ -51,25 +53,43 @@ class RangeStabilityFilter(IPairList):
         return (f"{self.name} - Filtering pairs with rate of change below "
                 f"{self._min_rate_of_change} over the last {plural(self._days, 'day')}.")
 
-    def _validate_pair(self, ticker: Dict) -> bool:
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
         Validate trading range
-        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param pairlist: pairlist to filter or sort
-        :return: True if the pair can stay, False if it should be removed
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new allowlist
+        """
+        needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
+
+        since_ms = int(arrow.utcnow()
+                       .floor('day')
+                       .shift(days=-self._days - 1)
+                       .float_timestamp) * 1000
+        # Get all candles
+        candles = {}
+        if needed_pairs:
+            candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms,
+                                                          cache=False)
+
+        if self._enabled:
+            for p in deepcopy(pairlist):
+                daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None
+                if not self._validate_pair_loc(p, daily_candles):
+                    pairlist.remove(p)
+        return pairlist
+
+    def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool:
+        """
+        Validate trading range
+        :param pair: Pair that's currently validated
+        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :return: True if the pair can stay, false if it should be removed
         """
-        pair = ticker['symbol']
         # Check symbol in cache
         if pair in self._pair_cache:
             return self._pair_cache[pair]
 
-        since_ms = int(arrow.utcnow()
-                       .floor('day')
-                       .shift(days=-self._days)
-                       .float_timestamp) * 1000
-
-        daily_candles = self._exchange.get_historic_ohlcv_as_df(pair=pair,
-                                                                timeframe='1d',
-                                                                since_ms=since_ms)
         result = False
         if daily_candles is not None and not daily_candles.empty:
             highest_high = daily_candles['high'].max()
@@ -79,7 +99,7 @@ class RangeStabilityFilter(IPairList):
                 result = True
             else:
                 self.log_once(f"Removed {pair} from whitelist, because rate of change "
-                              f"over {plural(self._days, 'day')} is {pct_change:.3f}, "
+                              f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
                               f"which is below the threshold of {self._min_rate_of_change}.",
                               logger.info)
                 result = False

tests/conftest.py

@@ -1084,7 +1084,7 @@ def ohlcv_history_list():
 @pytest.fixture
 def ohlcv_history(ohlcv_history_list):
     return ohlcv_to_dataframe(ohlcv_history_list, "5m", pair="UNITTEST/BTC",
-                              fill_missing=True)
+                              fill_missing=True, drop_incomplete=False)
 
 
 @pytest.fixture

tests/exchange/test_exchange.py

@@ -1385,6 +1385,12 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None:
     pairs = [('IOTA/ETH', '5m'), ('XRP/ETH', '5m')]
     # empty dicts
     assert not exchange._klines
+    exchange.refresh_latest_ohlcv(pairs, cache=False)
+    # No caching
+    assert not exchange._klines
+    assert exchange._api_async.fetch_ohlcv.call_count == 2
+    exchange._api_async.fetch_ohlcv.reset_mock()
+
     exchange.refresh_latest_ohlcv(pairs)
 
     assert log_has(f'Refreshing candle (OHLCV) data for {len(pairs)} pairs', caplog)
@@ -1499,11 +1505,9 @@ def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog):
     assert exchange._klines
     assert exchange._api_async.fetch_ohlcv.call_count == 2
 
-    assert type(res) is list
+    assert type(res) is dict
-    assert len(res) == 2
+    assert len(res) == 1
     # Test that each is in list at least once as order is not guaranteed
-    assert type(res[0]) is tuple or type(res[1]) is tuple
-    assert type(res[0]) is TypeError or type(res[1]) is TypeError
     assert log_has("Error loading ETH/BTC. Result was [[]].", caplog)
     assert log_has("Async code raised an exception: TypeError", caplog)
 

tests/plugins/test_pairlist.py

@@ -353,11 +353,19 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
      "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']),
 ])
 def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers,
-                                      ohlcv_history_list, pairlists, base_currency,
+                                      ohlcv_history, pairlists, base_currency,
                                       whitelist_result, caplog) -> None:
     whitelist_conf['pairlists'] = pairlists
     whitelist_conf['stake_currency'] = base_currency
 
+    ohlcv_data = {
+        ('ETH/BTC', '1d'): ohlcv_history,
+        ('TKN/BTC', '1d'): ohlcv_history,
+        ('LTC/BTC', '1d'): ohlcv_history,
+        ('XRP/BTC', '1d'): ohlcv_history,
+        ('HOT/BTC', '1d'): ohlcv_history,
+    }
+
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
 
     if whitelist_result == 'static_in_the_middle':
@@ -374,7 +382,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
                           )
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
-        get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list),
+        refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
     )
 
     # Provide for PerformanceFilter's dependency
@@ -402,7 +410,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
 
         for pairlist in pairlists:
             if pairlist['method'] == 'AgeFilter' and pairlist['min_days_listed'] and \
-                    len(ohlcv_history_list) <= pairlist['min_days_listed']:
+                    len(ohlcv_history) <= pairlist['min_days_listed']:
                 assert log_has_re(r'^Removed .* from whitelist, because age .* is less than '
                                   r'.* day.*', caplog)
             if pairlist['method'] == 'PrecisionFilter' and whitelist_result:
@@ -575,8 +583,13 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick
         get_patched_freqtradebot(mocker, default_conf)
 
 
-def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history_list):
+def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history):
-
+    ohlcv_data = {
+        ('ETH/BTC', '1d'): ohlcv_history,
+        ('TKN/BTC', '1d'): ohlcv_history,
+        ('LTC/BTC', '1d'): ohlcv_history,
+        ('XRP/BTC', '1d'): ohlcv_history,
+    }
     mocker.patch.multiple('freqtrade.exchange.Exchange',
                           markets=PropertyMock(return_value=markets),
                           exchange_has=MagicMock(return_value=True),
@@ -584,18 +597,18 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o
                           )
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
-        get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list),
+        refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
     )
 
     freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter)
-    assert freqtrade.exchange.get_historic_ohlcv.call_count == 0
+    assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0
     freqtrade.pairlists.refresh_pairlist()
-    assert freqtrade.exchange.get_historic_ohlcv.call_count > 0
+    assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0
 
-    previous_call_count = freqtrade.exchange.get_historic_ohlcv.call_count
+    previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count
     freqtrade.pairlists.refresh_pairlist()
     # Should not have increased since first call.
-    assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count
+    assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count
 
 
 def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers):
@@ -625,7 +638,7 @@ def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers):
     (0.01, 5),
     (0.05, 0),  # Setting rate_of_change to 5% removes all pairs from the whitelist.
 ])
-def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, ohlcv_history_list,
+def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, ohlcv_history,
                                       min_rate_of_change, expected_length):
     default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
                                  {'method': 'RangeStabilityFilter', 'lookback_days': 2,
@@ -636,22 +649,30 @@ def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, oh
                           exchange_has=MagicMock(return_value=True),
                           get_tickers=tickers
                           )
+    ohlcv_data = {
+        ('ETH/BTC', '1d'): ohlcv_history,
+        ('TKN/BTC', '1d'): ohlcv_history,
+        ('LTC/BTC', '1d'): ohlcv_history,
+        ('XRP/BTC', '1d'): ohlcv_history,
+        ('HOT/BTC', '1d'): ohlcv_history,
+        ('BLK/BTC', '1d'): ohlcv_history,
+    }
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
-        get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list),
+        refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
     )
 
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    assert freqtrade.exchange.get_historic_ohlcv.call_count == 0
+    assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0
     freqtrade.pairlists.refresh_pairlist()
     assert len(freqtrade.pairlists.whitelist) == expected_length
-    assert freqtrade.exchange.get_historic_ohlcv.call_count > 0
+    assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0
 
-    previous_call_count = freqtrade.exchange.get_historic_ohlcv.call_count
+    previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count
     freqtrade.pairlists.refresh_pairlist()
     assert len(freqtrade.pairlists.whitelist) == expected_length
     # Should not have increased since first call.
-    assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count
+    assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count
 
 
 @pytest.mark.parametrize("pairlistconfig,desc_expected,exception_expected", [

tests/rpc/test_rpc_apiserver.py

@@ -870,7 +870,7 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets):
 def test_api_pair_candles(botclient, ohlcv_history):
     ftbot, client = botclient
     timeframe = '5m'
-    amount = 2
+    amount = 3
 
     # No pair
     rc = client_get(client,
@@ -910,8 +910,8 @@ def test_api_pair_candles(botclient, ohlcv_history):
     assert 'data_stop_ts' in rc.json
     assert rc.json['data_start'] == '2017-11-26 08:50:00+00:00'
     assert rc.json['data_start_ts'] == 1511686200000
-    assert rc.json['data_stop'] == '2017-11-26 08:55:00+00:00'
+    assert rc.json['data_stop'] == '2017-11-26 09:00:00+00:00'
-    assert rc.json['data_stop_ts'] == 1511686500000
+    assert rc.json['data_stop_ts'] == 1511686800000
     assert isinstance(rc.json['columns'], list)
     assert rc.json['columns'] == ['date', 'open', 'high',
                                   'low', 'close', 'volume', 'sma', 'buy', 'sell',
@@ -926,7 +926,10 @@ def test_api_pair_candles(botclient, ohlcv_history):
             [['2017-11-26 08:50:00', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869,
               None, 0, 0, 1511686200000, None, None],
              ['2017-11-26 08:55:00', 8.88e-05, 8.942e-05, 8.88e-05,
-                 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0, 1511686500000, 8.88e-05, None]
+                 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0, 1511686500000, 8.88e-05, None],
+             ['2017-11-26 09:00:00', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05,
+                 0.7039405, 8.885000000000002e-05, 0, 0, 1511686800000, None, None]
+
              ])
 
 

tests/strategy/test_interface.py

@@ -128,27 +128,29 @@ def test_assert_df_raise(default_conf, mocker, caplog, ohlcv_history):
 
 
 def test_assert_df(default_conf, mocker, ohlcv_history, caplog):
+    df_len = len(ohlcv_history) - 1
     # Ensure it's running when passed correctly
     _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
-                        ohlcv_history.loc[1, 'close'], ohlcv_history.loc[1, 'date'])
+                        ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[df_len, 'date'])
 
     with pytest.raises(StrategyError, match=r"Dataframe returned from strategy.*length\."):
         _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history) + 1,
-                            ohlcv_history.loc[1, 'close'], ohlcv_history.loc[1, 'date'])
+                            ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[df_len, 'date'])
 
     with pytest.raises(StrategyError,
                        match=r"Dataframe returned from strategy.*last close price\."):
         _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
-                            ohlcv_history.loc[1, 'close'] + 0.01, ohlcv_history.loc[1, 'date'])
+                            ohlcv_history.loc[df_len, 'close'] + 0.01,
+                            ohlcv_history.loc[df_len, 'date'])
     with pytest.raises(StrategyError,
                        match=r"Dataframe returned from strategy.*last date\."):
         _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
-                            ohlcv_history.loc[1, 'close'], ohlcv_history.loc[0, 'date'])
+                            ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[0, 'date'])
 
     _STRATEGY.disable_dataframe_checks = True
     caplog.clear()
     _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
-                        ohlcv_history.loc[1, 'close'], ohlcv_history.loc[0, 'date'])
+                        ohlcv_history.loc[2, 'close'], ohlcv_history.loc[0, 'date'])
     assert log_has_re(r"Dataframe returned from strategy.*last date\.", caplog)
     # reset to avoid problems in other tests due to test leakage
     _STRATEGY.disable_dataframe_checks = False