From 6744f8f052d66e1f7a49eff81c6ff57e65fec0cf Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 10 Jun 2020 01:22:55 +0300 Subject: [PATCH 1/4] Remove _load_async_markets --- freqtrade/exchange/exchange.py | 17 +++-------------- tests/exchange/test_exchange.py | 28 ---------------------------- 2 files changed, 3 insertions(+), 42 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 038fc22bc..773f7a0a4 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -256,32 +256,21 @@ class Exchange: "Please check your config.json") raise OperationalException(f'Exchange {name} does not provide a sandbox api') - def _load_async_markets(self, reload: bool = False) -> None: - try: - if self._api_async: - asyncio.get_event_loop().run_until_complete( - self._api_async.load_markets(reload=reload)) - - except ccxt.BaseError as e: - logger.warning('Could not load async markets. Reason: %s', e) - return - def _load_markets(self) -> None: - """ Initialize markets both sync and async """ + """ Initialize markets """ try: self._api.load_markets() - self._load_async_markets() self._last_markets_refresh = arrow.utcnow().timestamp except ccxt.BaseError as e: logger.warning('Unable to initialize markets. Reason: %s', e) def _reload_markets(self) -> None: - """Reload markets both sync and async, if refresh interval has passed""" + """Reload markets if refresh interval has passed""" # Check whether markets have to be reloaded if (self._last_markets_refresh > 0) and ( self._last_markets_refresh + self.markets_refresh_interval > arrow.utcnow().timestamp): - return None + return logger.debug("Performing scheduled market reload..") try: self._api.load_markets(reload=True) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 32163f696..31efc65ff 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -130,7 +130,6 @@ def test_init_exception(default_conf, mocker): def test_exchange_resolver(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=MagicMock())) - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_pairs') mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') @@ -318,19 +317,6 @@ def test_set_sandbox_exception(default_conf, mocker): exchange.set_sandbox(exchange._api, default_conf['exchange'], 'Logname') -def test__load_async_markets(default_conf, mocker, caplog): - exchange = get_patched_exchange(mocker, default_conf) - exchange._api_async.load_markets = get_mock_coro(None) - exchange._load_async_markets() - assert exchange._api_async.load_markets.call_count == 1 - caplog.set_level(logging.DEBUG) - - exchange._api_async.load_markets = Mock(side_effect=ccxt.BaseError("deadbeef")) - exchange._load_async_markets() - - assert log_has('Could not load async markets. Reason: deadbeef', caplog) - - def test__load_markets(default_conf, mocker, caplog): caplog.set_level(logging.INFO) api_mock = MagicMock() @@ -338,7 +324,6 @@ def test__load_markets(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_pairs') mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) assert log_has('Unable to initialize markets. Reason: SomeError', caplog) @@ -406,7 +391,6 @@ def test_validate_stake_currency(default_conf, stake_currency, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_pairs') mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') Exchange(default_conf) @@ -420,7 +404,6 @@ def test_validate_stake_currency_error(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_pairs') mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') with pytest.raises(OperationalException, match=r'XRP is not available as stake on .*' 'Available currencies are: BTC, ETH, USDT'): @@ -470,7 +453,6 @@ def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs d mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) @@ -483,7 +465,6 @@ def test_validate_pairs_not_available(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') with pytest.raises(OperationalException, match=r'not available'): Exchange(default_conf) @@ -498,7 +479,6 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'): Exchange(default_conf) @@ -517,7 +497,6 @@ def test_validate_pairs_restricted(default_conf, mocker, caplog): }) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) @@ -535,7 +514,6 @@ def test_validate_pairs_stakecompatibility(default_conf, mocker, caplog): }) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) @@ -551,7 +529,6 @@ def test_validate_pairs_stakecompatibility_downloaddata(default_conf, mocker, ca }) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) @@ -567,7 +544,6 @@ def test_validate_pairs_stakecompatibility_fail(default_conf, mocker, caplog): }) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') with pytest.raises(OperationalException, match=r"Stake-currency 'BTC' not compatible with.*"): @@ -742,7 +718,6 @@ def test_validate_required_startup_candles(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_pairs') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') @@ -1934,7 +1909,6 @@ def test_stoploss_order_unsupported_exchange(default_conf, mocker): def test_merge_ft_has_dict(default_conf, mocker): mocker.patch.multiple('freqtrade.exchange.Exchange', _init_ccxt=MagicMock(return_value=MagicMock()), - _load_async_markets=MagicMock(), validate_pairs=MagicMock(), validate_timeframes=MagicMock(), validate_stakecurrency=MagicMock() @@ -1968,7 +1942,6 @@ def test_merge_ft_has_dict(default_conf, mocker): def test_get_valid_pair_combination(default_conf, mocker, markets): mocker.patch.multiple('freqtrade.exchange.Exchange', _init_ccxt=MagicMock(return_value=MagicMock()), - _load_async_markets=MagicMock(), validate_pairs=MagicMock(), validate_timeframes=MagicMock(), markets=PropertyMock(return_value=markets)) @@ -2041,7 +2014,6 @@ def test_get_markets(default_conf, mocker, markets, expected_keys): mocker.patch.multiple('freqtrade.exchange.Exchange', _init_ccxt=MagicMock(return_value=MagicMock()), - _load_async_markets=MagicMock(), validate_pairs=MagicMock(), validate_timeframes=MagicMock(), markets=PropertyMock(return_value=markets)) From 7d451638a84e9d709291a4038cb06f760bfa9c80 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 10 Jun 2020 01:39:23 +0300 Subject: [PATCH 2/4] Make _reload_markets() public --- freqtrade/exchange/exchange.py | 4 ++-- freqtrade/freqtradebot.py | 4 ++-- tests/exchange/test_exchange.py | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 773f7a0a4..6802a6b9e 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -264,8 +264,8 @@ class Exchange: except ccxt.BaseError as e: logger.warning('Unable to initialize markets. Reason: %s', e) - def _reload_markets(self) -> None: - """Reload markets if refresh interval has passed""" + def reload_markets(self) -> None: + """Reload markets if refresh interval has passed """ # Check whether markets have to be reloaded if (self._last_markets_refresh > 0) and ( self._last_markets_refresh + self.markets_refresh_interval diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5104e4f95..8a66957c3 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -139,8 +139,8 @@ class FreqtradeBot: :return: True if one or more trades has been created or closed, False otherwise """ - # Check whether markets have to be reloaded - self.exchange._reload_markets() + # Check whether markets have to be reloaded and reload them when it's needed + self.exchange.reload_markets() # Query trades from persistence layer trades = Trade.get_open_trades() diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 31efc65ff..9914188b8 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -337,7 +337,7 @@ def test__load_markets(default_conf, mocker, caplog): assert ex.markets == expected_return -def test__reload_markets(default_conf, mocker, caplog): +def test_reload_markets(default_conf, mocker, caplog): caplog.set_level(logging.DEBUG) initial_markets = {'ETH/BTC': {}} @@ -356,17 +356,17 @@ def test__reload_markets(default_conf, mocker, caplog): assert exchange.markets == initial_markets # less than 10 minutes have passed, no reload - exchange._reload_markets() + exchange.reload_markets() assert exchange.markets == initial_markets # more than 10 minutes have passed, reload is executed exchange._last_markets_refresh = arrow.utcnow().timestamp - 15 * 60 - exchange._reload_markets() + exchange.reload_markets() assert exchange.markets == updated_markets assert log_has('Performing scheduled market reload..', caplog) -def test__reload_markets_exception(default_conf, mocker, caplog): +def test_reload_markets_exception(default_conf, mocker, caplog): caplog.set_level(logging.DEBUG) api_mock = MagicMock() @@ -375,7 +375,7 @@ def test__reload_markets_exception(default_conf, mocker, caplog): exchange = get_patched_exchange(mocker, default_conf, api_mock, id="binance") # less than 10 minutes have passed, no reload - exchange._reload_markets() + exchange.reload_markets() assert exchange._last_markets_refresh == 0 assert log_has_re(r"Could not reload markets.*", caplog) From 0067a3ab7ceb929b2ded3a7916a064f759f45c69 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 10 Jun 2020 06:30:29 +0300 Subject: [PATCH 3/4] Change logging level --- freqtrade/exchange/exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 6802a6b9e..e974655cb 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -190,7 +190,7 @@ class Exchange: def markets(self) -> Dict: """exchange ccxt markets""" if not self._api.markets: - logger.warning("Markets were not loaded. Loading them now..") + logger.info("Markets were not loaded. Loading them now..") self._load_markets() return self._api.markets From 37bc2d28ad4572234940689fa938a48732d11207 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 13 Jun 2020 13:34:29 +0300 Subject: [PATCH 4/4] Revert "Remove _load_async_markets" This reverts commit 6744f8f052d66e1f7a49eff81c6ff57e65fec0cf. --- freqtrade/exchange/exchange.py | 17 ++++++++++++++--- tests/exchange/test_exchange.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index e974655cb..bd44f56f2 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -256,21 +256,32 @@ class Exchange: "Please check your config.json") raise OperationalException(f'Exchange {name} does not provide a sandbox api') + def _load_async_markets(self, reload: bool = False) -> None: + try: + if self._api_async: + asyncio.get_event_loop().run_until_complete( + self._api_async.load_markets(reload=reload)) + + except ccxt.BaseError as e: + logger.warning('Could not load async markets. Reason: %s', e) + return + def _load_markets(self) -> None: - """ Initialize markets """ + """ Initialize markets both sync and async """ try: self._api.load_markets() + self._load_async_markets() self._last_markets_refresh = arrow.utcnow().timestamp except ccxt.BaseError as e: logger.warning('Unable to initialize markets. Reason: %s', e) def reload_markets(self) -> None: - """Reload markets if refresh interval has passed """ + """Reload markets both sync and async if refresh interval has passed """ # Check whether markets have to be reloaded if (self._last_markets_refresh > 0) and ( self._last_markets_refresh + self.markets_refresh_interval > arrow.utcnow().timestamp): - return + return None logger.debug("Performing scheduled market reload..") try: self._api.load_markets(reload=True) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 9914188b8..2b63eee23 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -130,6 +130,7 @@ def test_init_exception(default_conf, mocker): def test_exchange_resolver(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=MagicMock())) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_pairs') mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') @@ -317,6 +318,19 @@ def test_set_sandbox_exception(default_conf, mocker): exchange.set_sandbox(exchange._api, default_conf['exchange'], 'Logname') +def test__load_async_markets(default_conf, mocker, caplog): + exchange = get_patched_exchange(mocker, default_conf) + exchange._api_async.load_markets = get_mock_coro(None) + exchange._load_async_markets() + assert exchange._api_async.load_markets.call_count == 1 + caplog.set_level(logging.DEBUG) + + exchange._api_async.load_markets = Mock(side_effect=ccxt.BaseError("deadbeef")) + exchange._load_async_markets() + + assert log_has('Could not load async markets. Reason: deadbeef', caplog) + + def test__load_markets(default_conf, mocker, caplog): caplog.set_level(logging.INFO) api_mock = MagicMock() @@ -324,6 +338,7 @@ def test__load_markets(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_pairs') mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) assert log_has('Unable to initialize markets. Reason: SomeError', caplog) @@ -391,6 +406,7 @@ def test_validate_stake_currency(default_conf, stake_currency, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_pairs') mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') Exchange(default_conf) @@ -404,6 +420,7 @@ def test_validate_stake_currency_error(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_pairs') mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') with pytest.raises(OperationalException, match=r'XRP is not available as stake on .*' 'Available currencies are: BTC, ETH, USDT'): @@ -453,6 +470,7 @@ def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs d mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) @@ -465,6 +483,7 @@ def test_validate_pairs_not_available(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') with pytest.raises(OperationalException, match=r'not available'): Exchange(default_conf) @@ -479,6 +498,7 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'): Exchange(default_conf) @@ -497,6 +517,7 @@ def test_validate_pairs_restricted(default_conf, mocker, caplog): }) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) @@ -514,6 +535,7 @@ def test_validate_pairs_stakecompatibility(default_conf, mocker, caplog): }) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) @@ -529,6 +551,7 @@ def test_validate_pairs_stakecompatibility_downloaddata(default_conf, mocker, ca }) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) @@ -544,6 +567,7 @@ def test_validate_pairs_stakecompatibility_fail(default_conf, mocker, caplog): }) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') with pytest.raises(OperationalException, match=r"Stake-currency 'BTC' not compatible with.*"): @@ -718,6 +742,7 @@ def test_validate_required_startup_candles(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_pairs') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') @@ -1909,6 +1934,7 @@ def test_stoploss_order_unsupported_exchange(default_conf, mocker): def test_merge_ft_has_dict(default_conf, mocker): mocker.patch.multiple('freqtrade.exchange.Exchange', _init_ccxt=MagicMock(return_value=MagicMock()), + _load_async_markets=MagicMock(), validate_pairs=MagicMock(), validate_timeframes=MagicMock(), validate_stakecurrency=MagicMock() @@ -1942,6 +1968,7 @@ def test_merge_ft_has_dict(default_conf, mocker): def test_get_valid_pair_combination(default_conf, mocker, markets): mocker.patch.multiple('freqtrade.exchange.Exchange', _init_ccxt=MagicMock(return_value=MagicMock()), + _load_async_markets=MagicMock(), validate_pairs=MagicMock(), validate_timeframes=MagicMock(), markets=PropertyMock(return_value=markets)) @@ -2014,6 +2041,7 @@ def test_get_markets(default_conf, mocker, markets, expected_keys): mocker.patch.multiple('freqtrade.exchange.Exchange', _init_ccxt=MagicMock(return_value=MagicMock()), + _load_async_markets=MagicMock(), validate_pairs=MagicMock(), validate_timeframes=MagicMock(), markets=PropertyMock(return_value=markets))