From a486b1d01c631fe0b4c0334fc3e6e7fe61e84206 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jul 2018 20:25:10 +0200 Subject: [PATCH 1/6] Use Dict instead of tuplelist, run in _process --- freqtrade/freqtradebot.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 93977ab16..e17498b51 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -149,6 +149,10 @@ class FreqtradeBot(object): final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list self.config['exchange']['pair_whitelist'] = final_list + datatups = asyncio.get_event_loop().run_until_complete( + self.exchange.async_get_tickers_history(final_list, self.strategy.ticker_interval)) + self._klines = {pair: data for (pair, data) in datatups} + # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -337,25 +341,25 @@ class FreqtradeBot(object): if not whitelist: raise DependencyException('No currency pairs in whitelist') - + # fetching kline history for all pairs asynchronously and wait till all done - data = asyncio.get_event_loop().run_until_complete(self.exchange.async_get_tickers_history(whitelist, self.strategy.ticker_interval)) - + # data = asyncio.get_event_loop().run_until_complete(self.exchange.async_get_tickers_history(whitelist, self.strategy.ticker_interval)) + # list of pairs having buy signals buy_pairs = [] - # running get_signal on historical data fetched + # running get_signal on historical data fetched # to find buy signals - for _pair, thistory in data: + for _pair, thistory in self._klines.items(): (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) - if buy and not sell: + if buy and not sell: buy_pairs.append(_pair) - # If there is at least one buy signal then - # Go ahead and buy the first pair + # If there is at least one buy signal then + # Go ahead and buy the first pair if buy_pairs: return self.execute_buy(buy_pairs[0], stake_amount) - + return False def execute_buy(self, pair: str, stake_amount: float) -> bool: @@ -518,7 +522,8 @@ class FreqtradeBot(object): (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): - ticker = self.exchange.get_ticker_history(trade.pair, self.strategy.ticker_interval) + # ticker = self.exchange.get_ticker_history(trade.pair, self.strategy.ticker_interval) + ticker = self._klines[trade.pair] (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, ticker) From 31870abd251f1cf19fd0acf8890fb69443445006 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jul 2018 20:43:32 +0200 Subject: [PATCH 2/6] Refactor async-refresh to it's own function --- freqtrade/exchange/__init__.py | 28 +++++++++++++++++----------- freqtrade/freqtradebot.py | 21 ++++----------------- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index abbc9808b..8c5793b83 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -2,7 +2,7 @@ """ Cryptocurrency Exchanges support """ import logging from random import randint -from typing import List, Dict, Any, Optional +from typing import List, Dict, Tuple, Any, Optional from datetime import datetime from math import floor, ceil @@ -95,7 +95,7 @@ class Exchange(object): 'secret': exchange_config.get('secret'), 'password': exchange_config.get('password'), 'uid': exchange_config.get('uid', ''), - #'enableRateLimit': True, + # 'enableRateLimit': True, 'enableRateLimit': False, }) except (KeyError, AttributeError): @@ -334,23 +334,23 @@ class Exchange(object): logger.info("returning cached ticker-data for %s", pair) return self._cached_ticker[pair] - async def async_get_tickers_history(self, pairs, tick_interval): - # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ? - #loop = asyncio.new_event_loop() - #asyncio.set_event_loop(loop) - input_coroutines = [self.async_get_ticker_history(symbol, tick_interval) for symbol in pairs] + # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ? + # loop = asyncio.new_event_loop() + # asyncio.set_event_loop(loop) + input_coroutines = [self.async_get_ticker_history( + symbol, tick_interval) for symbol in pairs] tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) - #await self._api_async.close() + # await self._api_async.close() return tickers async def async_get_ticker_history(self, pair: str, tick_interval: str, - since_ms: Optional[int] = None) -> List[Dict]: + since_ms: Optional[int] = None) -> Tuple[str, List]: try: # fetch ohlcv asynchronously - print("fetching %s ..." % pair) + logger.debug("fetching %s ...", pair) data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) - print("done fetching %s ..." % pair) + logger.debug("done fetching %s ...", pair) return pair, data except ccxt.NotSupported as e: @@ -363,6 +363,12 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(f'Could not fetch ticker data. Msg: {e}') + def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> Dict: + logger.debug("Refreshing klines for %d pairs", len(pair_list)) + datatups = asyncio.get_event_loop().run_until_complete( + self.async_get_tickers_history(pair_list, ticker_interval)) + return {pair: data for (pair, data) in datatups} + @retrier def get_ticker_history(self, pair: str, tick_interval: str, since_ms: Optional[int] = None) -> List[Dict]: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e17498b51..eaa2184ac 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -8,7 +8,6 @@ import time import traceback from datetime import datetime from typing import Any, Callable, Dict, List, Optional -import asyncio import arrow import requests @@ -149,9 +148,7 @@ class FreqtradeBot(object): final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list self.config['exchange']['pair_whitelist'] = final_list - datatups = asyncio.get_event_loop().run_until_complete( - self.exchange.async_get_tickers_history(final_list, self.strategy.ticker_interval)) - self._klines = {pair: data for (pair, data) in datatups} + self._klines = self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval) # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -306,13 +303,7 @@ class FreqtradeBot(object): amount_reserve_percent += self.strategy.stoploss # it should not be more than 50% amount_reserve_percent = max(amount_reserve_percent, 0.5) - return min(min_stake_amounts)/amount_reserve_percent - - async def async_get_tickers(self, exchange, pairs): - input_coroutines = [exchange.async_get_ticker_history(symbol, self.strategy.ticker_interval) for symbol in pairs] - tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) - return tickers - #await exchange.close() + return min(min_stake_amounts) / amount_reserve_percent def create_trade(self) -> bool: """ @@ -341,17 +332,13 @@ class FreqtradeBot(object): if not whitelist: raise DependencyException('No currency pairs in whitelist') - - # fetching kline history for all pairs asynchronously and wait till all done - # data = asyncio.get_event_loop().run_until_complete(self.exchange.async_get_tickers_history(whitelist, self.strategy.ticker_interval)) - # list of pairs having buy signals buy_pairs = [] # running get_signal on historical data fetched # to find buy signals - for _pair, thistory in self._klines.items(): - (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) + for _pair in whitelist: + (buy, sell) = self.strategy.get_signal(_pair, interval, self._klines[_pair]) if buy and not sell: buy_pairs.append(_pair) From b45d465ed833264aaf1832b8f048fbbd4c6fc306 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jul 2018 20:50:59 +0200 Subject: [PATCH 3/6] init _klines properly --- freqtrade/freqtradebot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index eaa2184ac..4313b310d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -55,6 +55,7 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self._init_modules() + self._klines = {} def _init_modules(self) -> None: """ @@ -338,7 +339,7 @@ class FreqtradeBot(object): # running get_signal on historical data fetched # to find buy signals for _pair in whitelist: - (buy, sell) = self.strategy.get_signal(_pair, interval, self._klines[_pair]) + (buy, sell) = self.strategy.get_signal(_pair, interval, self._klines.get(_pair)) if buy and not sell: buy_pairs.append(_pair) From 52065178e16ad2e84a1774e73e6de5b878f3dbe1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jul 2018 20:53:32 +0200 Subject: [PATCH 4/6] use .get all the time --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4313b310d..365d145ca 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -511,7 +511,7 @@ class FreqtradeBot(object): experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): # ticker = self.exchange.get_ticker_history(trade.pair, self.strategy.ticker_interval) - ticker = self._klines[trade.pair] + ticker = self._klines.get(trade.pair) (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, ticker) From 12417cc303715244dc1701d13b5d5ed77d12b6ed Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jul 2018 20:54:51 +0200 Subject: [PATCH 5/6] fix tests --- freqtrade/tests/test_freqtradebot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 69f349107..fc84464e4 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -44,6 +44,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: """ freqtrade.strategy.get_signal = lambda e, s, t: value freqtrade.exchange.get_ticker_history = lambda p, i: None + freqtrade.exchange.refresh_tickers = lambda pl, i: {} def patch_RPCManager(mocker) -> MagicMock: From 136442245c850c83bb314230306a27ef1924992d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jul 2018 21:01:44 +0200 Subject: [PATCH 6/6] Add todo's and dockstring --- freqtrade/exchange/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 8c5793b83..8e88a7f5f 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -364,6 +364,12 @@ class Exchange(object): raise OperationalException(f'Could not fetch ticker data. Msg: {e}') def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> Dict: + """ + Refresh tickers asyncronously and return the result. + """ + # TODO: maybe add since_ms to use async in the download-script? + # TODO: only refresh once per interval ? *may require this to move to freqtradebot.py + # TODO@ Add tests for this and the async stuff above logger.debug("Refreshing klines for %d pairs", len(pair_list)) datatups = asyncio.get_event_loop().run_until_complete( self.async_get_tickers_history(pair_list, ticker_interval))