Merge pull request #7263 from freqtrade/okx_cache_tiers

Okx cache tiers

freqtrade/exchange/exchange.py

@@ -17,6 +17,7 @@ import ccxt
 import ccxt.async_support as ccxt_async
 from cachetools import TTLCache
 from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, decimal_to_precision
+from dateutil import parser
 from pandas import DataFrame
 
 from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell,
@@ -30,7 +31,8 @@ from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGE
                                        EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED,
                                        SUPPORTED_EXCHANGES, remove_credentials, retrier,
                                        retrier_async)
-from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2
+from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_json,
+                            safe_value_fallback2)
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.util import FtPrecise
 
@@ -2207,6 +2209,7 @@ class Exchange:
 
     @retrier_async
     async def get_market_leverage_tiers(self, symbol: str) -> Tuple[str, List[Dict]]:
+        """ Leverage tiers per symbol """
         try:
             tier = await self._api_async.fetch_market_leverage_tiers(symbol)
             return symbol, tier
@@ -2238,12 +2241,21 @@ class Exchange:
 
                 tiers: Dict[str, List[Dict]] = {}
 
-                # Be verbose here, as this delays startup by ~1 minute.
+                tiers_cached = self.load_cached_leverage_tiers(self._config['stake_currency'])
-                logger.info(
+                if tiers_cached:
-                    f"Initializing leverage_tiers for {len(symbols)} markets. "
+                    tiers = tiers_cached
-                    "This will take about a minute.")
 
-                coros = [self.get_market_leverage_tiers(symbol) for symbol in sorted(symbols)]
+                coros = [
+                    self.get_market_leverage_tiers(symbol)
+                    for symbol in sorted(symbols) if symbol not in tiers]
+
+                # Be verbose here, as this delays startup by ~1 minute.
+                if coros:
+                    logger.info(
+                        f"Initializing leverage_tiers for {len(symbols)} markets. "
+                        "This will take about a minute.")
+                else:
+                    logger.info("Using cached leverage_tiers.")
 
                 async def gather_results():
                     return await asyncio.gather(*input_coro, return_exceptions=True)
@@ -2255,7 +2267,8 @@ class Exchange:
 
                     for symbol, res in results:
                         tiers[symbol] = res
-
+                if len(coros) > 0:
+                    self.cache_leverage_tiers(tiers, self._config['stake_currency'])
                 logger.info(f"Done initializing {len(symbols)} markets.")
 
                 return tiers
@@ -2264,6 +2277,30 @@ class Exchange:
         else:
             return {}
 
+    def cache_leverage_tiers(self, tiers: Dict[str, List[Dict]], stake_currency: str) -> None:
+
+        filename = self._config['datadir'] / "futures" / f"leverage_tiers_{stake_currency}.json"
+        if not filename.parent.is_dir():
+            filename.parent.mkdir(parents=True)
+        data = {
+            "updated": datetime.now(timezone.utc),
+            "data": tiers,
+        }
+        file_dump_json(filename, data)
+
+    def load_cached_leverage_tiers(self, stake_currency: str) -> Optional[Dict[str, List[Dict]]]:
+        filename = self._config['datadir'] / "futures" / f"leverage_tiers_{stake_currency}.json"
+        if filename.is_file():
+            tiers = file_load_json(filename)
+            updated = tiers.get('updated')
+            if updated:
+                updated_dt = parser.parse(updated)
+                if updated_dt < datetime.now(timezone.utc) - timedelta(days=1):
+                    logger.info("Cached leverage tiers are outdated. Will update.")
+                    return None
+            return tiers['data']
+        return None
+
     def fill_leverage_tiers(self) -> None:
         """
         Assigns property _leverage_tiers to a dictionary of information about the leverage

tests/exchange/test_ccxt_compat.py

@@ -137,6 +137,10 @@ def exchange_futures(request, exchange_conf, class_mocker):
             'freqtrade.exchange.binance.Binance.fill_leverage_tiers')
         class_mocker.patch('freqtrade.exchange.exchange.Exchange.fetch_trading_fees')
         class_mocker.patch('freqtrade.exchange.okx.Okx.additional_exchange_init')
+        class_mocker.patch('freqtrade.exchange.exchange.Exchange.load_cached_leverage_tiers',
+                           return_value=None)
+        class_mocker.patch('freqtrade.exchange.exchange.Exchange.cache_leverage_tiers')
+
         exchange = ExchangeResolver.load_exchange(
             request.param, exchange_conf, validate=True, load_leverage_tiers=True)
 

tests/exchange/test_exchange.py

@@ -4791,6 +4791,20 @@ def test_load_leverage_tiers(mocker, default_conf, leverage_tiers, exchange_name
     )
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize('exchange_name', EXCHANGES)
+async def test_get_market_leverage_tiers(mocker, default_conf, exchange_name):
+    default_conf['exchange']['name'] = exchange_name
+    await async_ccxt_exception(
+        mocker,
+        default_conf,
+        MagicMock(),
+        "get_market_leverage_tiers",
+        "fetch_market_leverage_tiers",
+        symbol='BTC/USDT:USDT'
+    )
+
+
 def test_parse_leverage_tier(mocker, default_conf):
     exchange = get_patched_exchange(mocker, default_conf)
 

tests/exchange/test_okx.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta, timezone
+from pathlib import Path
 from unittest.mock import MagicMock, PropertyMock
 
 import pytest
@@ -6,7 +7,7 @@ import pytest
 from freqtrade.enums import MarginMode, TradingMode
 from freqtrade.enums.candletype import CandleType
 from freqtrade.exchange.exchange import timeframe_to_minutes
-from tests.conftest import get_mock_coro, get_patched_exchange
+from tests.conftest import get_mock_coro, get_patched_exchange, log_has
 from tests.exchange.test_exchange import ccxt_exceptionhandlers
 
 
@@ -267,7 +268,10 @@ def test_additional_exchange_init_okx(default_conf, mocker):
                            "additional_exchange_init", "fetch_accounts")
 
 
-def test_load_leverage_tiers_okx(default_conf, mocker, markets):
+def test_load_leverage_tiers_okx(default_conf, mocker, markets, tmpdir, caplog, time_machine):
+
+    default_conf['datadir'] = Path(tmpdir)
+    # fd_mock = mocker.patch('freqtrade.exchange.exchange.file_dump_json')
     api_mock = MagicMock()
     type(api_mock).has = PropertyMock(return_value={
         'fetchLeverageTiers': False,
@@ -455,3 +459,21 @@ def test_load_leverage_tiers_okx(default_conf, mocker, markets):
             },
         ],
     }
+    filename = (default_conf['datadir'] /
+                f"futures/leverage_tiers_{default_conf['stake_currency']}.json")
+    assert filename.is_file()
+
+    logmsg = 'Cached leverage tiers are outdated. Will update.'
+    assert not log_has(logmsg, caplog)
+
+    api_mock.fetch_market_leverage_tiers.reset_mock()
+
+    exchange.load_leverage_tiers()
+    assert not log_has(logmsg, caplog)
+
+    api_mock.fetch_market_leverage_tiers.call_count == 0
+    # 2 day passes ...
+    time_machine.move_to(datetime.now() + timedelta(days=2))
+    exchange.load_leverage_tiers()
+
+    assert log_has(logmsg, caplog)