Merge pull request #4367 from freqtrade/fix/4181

ohlcv_candle_limit per timeframe

freqtrade/exchange/bittrex.py

@@ -19,5 +19,11 @@ class Bittrex(Exchange):
     """
 
     _ft_has: Dict = {
+        "ohlcv_candle_limit_per_timeframe": {
+            '1m': 1440,
+            '5m': 288,
+            '1h': 744,
+            '1d': 365,
+        },
         "l2_limit_range": [1, 25, 500],
     }

freqtrade/exchange/exchange.py

@@ -101,7 +101,6 @@ class Exchange:
             logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has)
 
         # Assign this directly for easy access
-        self._ohlcv_candle_limit = self._ft_has['ohlcv_candle_limit']
         self._ohlcv_partial_candle = self._ft_has['ohlcv_partial_candle']
 
         self._trades_pagination = self._ft_has['trades_pagination']
@@ -137,7 +136,8 @@ class Exchange:
                 self.validate_pairs(config['exchange']['pair_whitelist'])
             self.validate_ordertypes(config.get('order_types', {}))
             self.validate_order_time_in_force(config.get('order_time_in_force', {}))
-            self.validate_required_startup_candles(config.get('startup_candle_count', 0))
+            self.validate_required_startup_candles(config.get('startup_candle_count', 0),
+                                                   config.get('timeframe', ''))
 
         # Converts the interval provided in minutes in config to seconds
         self.markets_refresh_interval: int = exchange_config.get(
@@ -198,11 +198,6 @@ class Exchange:
     def timeframes(self) -> List[str]:
         return list((self._api.timeframes or {}).keys())
 
-    @property
-    def ohlcv_candle_limit(self) -> int:
-        """exchange ohlcv candle limit"""
-        return int(self._ohlcv_candle_limit)
-
     @property
     def markets(self) -> Dict:
         """exchange ccxt markets"""
@@ -216,6 +211,17 @@ class Exchange:
         """exchange ccxt precisionMode"""
         return self._api.precisionMode
 
+    def ohlcv_candle_limit(self, timeframe: str) -> int:
+        """
+        Exchange ohlcv candle limit
+        Uses ohlcv_candle_limit_per_timeframe if the exchange has different limts
+        per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit
+        :param timeframe: Timeframe to check
+        :return: Candle limit as integer
+        """
+        return int(self._ft_has.get('ohlcv_candle_limit_per_timeframe', {}).get(
+            timeframe, self._ft_has.get('ohlcv_candle_limit')))
+
     def get_markets(self, base_currencies: List[str] = None, quote_currencies: List[str] = None,
                     pairs_only: bool = False, active_only: bool = False) -> Dict[str, Any]:
         """
@@ -428,15 +434,16 @@ class Exchange:
             raise OperationalException(
                 f'Time in force policies are not supported for {self.name} yet.')
 
-    def validate_required_startup_candles(self, startup_candles: int) -> None:
+    def validate_required_startup_candles(self, startup_candles: int, timeframe: str) -> None:
         """
-        Checks if required startup_candles is more than ohlcv_candle_limit.
+        Checks if required startup_candles is more than ohlcv_candle_limit().
         Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default.
         """
-        if startup_candles + 5 > self._ft_has['ohlcv_candle_limit']:
+        candle_limit = self.ohlcv_candle_limit(timeframe)
+        if startup_candles + 5 > candle_limit:
             raise OperationalException(
                 f"This strategy requires {startup_candles} candles to start. "
-                f"{self.name} only provides {self._ft_has['ohlcv_candle_limit']}.")
+                f"{self.name} only provides {candle_limit} for {timeframe}.")
 
     def exchange_has(self, endpoint: str) -> bool:
         """
@@ -721,7 +728,7 @@ class Exchange:
         """
         Get candle history using asyncio and returns the list of candles.
         Handles all async work for this.
-        Async over one pair, assuming we get `self._ohlcv_candle_limit` candles per call.
+        Async over one pair, assuming we get `self.ohlcv_candle_limit()` candles per call.
         :param pair: Pair to download
         :param timeframe: Timeframe to get data for
         :param since_ms: Timestamp in milliseconds to get history from
@@ -751,7 +758,7 @@ class Exchange:
         Download historic ohlcv
         """
 
-        one_call = timeframe_to_msecs(timeframe) * self._ohlcv_candle_limit
+        one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe)
         logger.debug(
             "one_call: %s msecs (%s)",
             one_call,
@@ -853,7 +860,7 @@ class Exchange:
 
             data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe,
                                                      since=since_ms,
-                                                     limit=self._ohlcv_candle_limit)
+                                                     limit=self.ohlcv_candle_limit(timeframe))
 
             # Some exchanges sort OHLCV in ASC order and others in DESC.
             # Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last)
@@ -1026,7 +1033,7 @@ class Exchange:
         """
         Get trade history data using asyncio.
         Handles all async work and returns the list of candles.
-        Async over one pair, assuming we get `self._ohlcv_candle_limit` candles per call.
+        Async over one pair, assuming we get `self.ohlcv_candle_limit()` candles per call.
         :param pair: Pair to download
         :param since: Timestamp in milliseconds to get history from
         :param until: Timestamp in milliseconds. Defaults to current timestamp if not defined.

freqtrade/plugins/pairlist/AgeFilter.py

@@ -30,10 +30,10 @@ class AgeFilter(IPairList):
 
         if self._min_days_listed < 1:
             raise OperationalException("AgeFilter requires min_days_listed to be >= 1")
-        if self._min_days_listed > exchange.ohlcv_candle_limit:
+        if self._min_days_listed > exchange.ohlcv_candle_limit('1d'):
             raise OperationalException("AgeFilter requires min_days_listed to not exceed "
                                        "exchange max request size "
-                                       f"({exchange.ohlcv_candle_limit})")
+                                       f"({exchange.ohlcv_candle_limit('1d')})")
 
     @property
     def needstickers(self) -> bool:

freqtrade/plugins/pairlist/rangestabilityfilter.py

@@ -32,10 +32,10 @@ class RangeStabilityFilter(IPairList):
 
         if self._days < 1:
             raise OperationalException("RangeStabilityFilter requires lookback_days to be >= 1")
-        if self._days > exchange.ohlcv_candle_limit:
+        if self._days > exchange.ohlcv_candle_limit('1d'):
             raise OperationalException("RangeStabilityFilter requires lookback_days to not "
                                        "exceed exchange max request size "
-                                       f"({exchange.ohlcv_candle_limit})")
+                                       f"({exchange.ohlcv_candle_limit('1d')})")
 
     @property
     def needstickers(self) -> bool:

tests/exchange/test_ccxt_compat.py

@@ -5,10 +5,12 @@ However, these tests should give a good idea to determine if a new exchange is
 suitable to run with freqtrade.
 """
 
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 
 import pytest
 
+from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
 from freqtrade.resolvers.exchange_resolver import ExchangeResolver
 from tests.conftest import get_default_conf
 
@@ -122,7 +124,10 @@ class TestCCXTExchange():
         assert len(ohlcv[pair_tf]) == len(exchange.klines(pair_tf))
         # assert len(exchange.klines(pair_tf)) > 200
         # Assume 90% uptime ...
-        assert len(exchange.klines(pair_tf)) > exchange._ohlcv_candle_limit * 0.90
+        assert len(exchange.klines(pair_tf)) > exchange.ohlcv_candle_limit(timeframe) * 0.90
+        # Check if last-timeframe is within the last 2 intervals
+        now = datetime.now(timezone.utc) - timedelta(minutes=(timeframe_to_minutes(timeframe) * 2))
+        assert exchange.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now)
 
     # TODO: tests fetch_trades (?)
 

tests/exchange/test_exchange.py

@@ -1417,7 +1417,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name):
     exchange._async_get_candle_history = Mock(wraps=mock_candle_hist)
     # one_call calculation * 1.8 should do 2 calls
 
-    since = 5 * 60 * exchange._ft_has['ohlcv_candle_limit'] * 1.8
+    since = 5 * 60 * exchange.ohlcv_candle_limit('5m') * 1.8
     ret = exchange.get_historic_ohlcv(pair, "5m", int((
         arrow.utcnow().int_timestamp - since) * 1000))
 
@@ -1473,7 +1473,7 @@ def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name):
     exchange._async_get_candle_history = Mock(wraps=mock_candle_hist)
     # one_call calculation * 1.8 should do 2 calls
 
-    since = 5 * 60 * exchange._ft_has['ohlcv_candle_limit'] * 1.8
+    since = 5 * 60 * exchange.ohlcv_candle_limit('5m') * 1.8
     ret = exchange.get_historic_ohlcv_as_df(pair, "5m", int((
         arrow.utcnow().int_timestamp - since) * 1000))
 
@@ -2418,6 +2418,19 @@ def test_get_markets_error(default_conf, mocker):
         ex.get_markets('LTC', 'USDT', True, False)
 
 
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+def test_ohlcv_candle_limit(default_conf, mocker, exchange_name):
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    timeframes = ('1m', '5m', '1h')
+    expected = exchange._ft_has['ohlcv_candle_limit']
+    for timeframe in timeframes:
+        if 'ohlcv_candle_limit_per_timeframe' in exchange._ft_has:
+            expected = exchange._ft_has['ohlcv_candle_limit_per_timeframe'][timeframe]
+            # This should only run for bittrex
+            assert exchange_name == 'bittrex'
+        assert exchange.ohlcv_candle_limit(timeframe) == expected
+
+
 def test_timeframe_to_minutes():
     assert timeframe_to_minutes("5m") == 5
     assert timeframe_to_minutes("10m") == 10