From c8f125dbb99dd2042898c4509064ed692379abee Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 31 Jul 2018 12:47:32 +0200 Subject: [PATCH 001/175] ccxt async POC --- freqtrade/exchange/__init__.py | 42 ++++++++++++++++++++++++++++++---- freqtrade/freqtradebot.py | 33 +++++++++++++++++++++----- 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 972ff49ca..6b9f000eb 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -6,7 +6,9 @@ from typing import List, Dict, Any, Optional from datetime import datetime import ccxt +import ccxt.async_support as ccxt_async import arrow +import asyncio from freqtrade import constants, OperationalException, DependencyException, TemporaryError @@ -44,6 +46,7 @@ class Exchange(object): # Current selected exchange _api: ccxt.Exchange = None + _api_async: ccxt_async.Exchange = None _conf: Dict = {} _cached_ticker: Dict[str, Any] = {} @@ -64,6 +67,7 @@ class Exchange(object): exchange_config = config['exchange'] self._api = self._init_ccxt(exchange_config) + self._api_async = self._init_ccxt(exchange_config, ccxt_async) logger.info('Using Exchange "%s"', self.name) @@ -74,7 +78,7 @@ class Exchange(object): # Check if timeframe is available self.validate_timeframes(config['ticker_interval']) - def _init_ccxt(self, exchange_config: dict) -> ccxt.Exchange: + def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid ccxt instance. @@ -82,15 +86,16 @@ class Exchange(object): # Find matching class for the given exchange name name = exchange_config['name'] - if name not in ccxt.exchanges: + if name not in ccxt_module.exchanges: raise OperationalException(f'Exchange {name} is not supported') try: - api = getattr(ccxt, name.lower())({ + api = getattr(ccxt_module, name.lower())({ 'apiKey': exchange_config.get('key'), 'secret': exchange_config.get('secret'), 'password': exchange_config.get('password'), 'uid': exchange_config.get('uid', ''), - 'enableRateLimit': True, + #'enableRateLimit': True, + 'enableRateLimit': False, }) except (KeyError, AttributeError): raise OperationalException(f'Exchange {name} is not supported') @@ -286,6 +291,35 @@ class Exchange(object): logger.info("returning cached ticker-data for %s", pair) return self._cached_ticker[pair] + + async def async_get_tickers_history(self, pairs, tick_interval): + # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ? + #loop = asyncio.new_event_loop() + #asyncio.set_event_loop(loop) + input_coroutines = [self.async_get_ticker_history(symbol, tick_interval) for symbol in pairs] + tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) + #await self._api_async.close() + return tickers + + async def async_get_ticker_history(self, pair: str, tick_interval: str, + since_ms: Optional[int] = None) -> List[Dict]: + try: + # fetch ohlcv asynchronously + print("fetching %s ..." % pair) + data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) + print("done fetching %s ..." % pair) + return pair, data + + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching historical candlestick data.' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(f'Could not fetch ticker data. Msg: {e}') + @retrier def get_ticker_history(self, pair: str, tick_interval: str, since_ms: Optional[int] = None) -> List[Dict]: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 46fbb3a38..93977ab16 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -8,11 +8,14 @@ import time import traceback from datetime import datetime from typing import Any, Callable, Dict, List, Optional +import asyncio import arrow import requests + from cachetools import TTLCache, cached + from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange @@ -301,6 +304,12 @@ class FreqtradeBot(object): amount_reserve_percent = max(amount_reserve_percent, 0.5) return min(min_stake_amounts)/amount_reserve_percent + async def async_get_tickers(self, exchange, pairs): + input_coroutines = [exchange.async_get_ticker_history(symbol, self.strategy.ticker_interval) for symbol in pairs] + tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) + return tickers + #await exchange.close() + def create_trade(self) -> bool: """ Checks the implemented trading indicator(s) for a randomly picked pair, @@ -328,13 +337,25 @@ class FreqtradeBot(object): if not whitelist: raise DependencyException('No currency pairs in whitelist') - # Pick pair based on buy signals - for _pair in whitelist: - thistory = self.exchange.get_ticker_history(_pair, interval) - (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) + + # fetching kline history for all pairs asynchronously and wait till all done + data = asyncio.get_event_loop().run_until_complete(self.exchange.async_get_tickers_history(whitelist, self.strategy.ticker_interval)) + + # list of pairs having buy signals + buy_pairs = [] - if buy and not sell: - return self.execute_buy(_pair, stake_amount) + # running get_signal on historical data fetched + # to find buy signals + for _pair, thistory in data: + (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) + if buy and not sell: + buy_pairs.append(_pair) + + # If there is at least one buy signal then + # Go ahead and buy the first pair + if buy_pairs: + return self.execute_buy(buy_pairs[0], stake_amount) + return False def execute_buy(self, pair: str, stake_amount: float) -> bool: From a486b1d01c631fe0b4c0334fc3e6e7fe61e84206 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jul 2018 20:25:10 +0200 Subject: [PATCH 002/175] Use Dict instead of tuplelist, run in _process --- freqtrade/freqtradebot.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 93977ab16..e17498b51 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -149,6 +149,10 @@ class FreqtradeBot(object): final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list self.config['exchange']['pair_whitelist'] = final_list + datatups = asyncio.get_event_loop().run_until_complete( + self.exchange.async_get_tickers_history(final_list, self.strategy.ticker_interval)) + self._klines = {pair: data for (pair, data) in datatups} + # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -337,25 +341,25 @@ class FreqtradeBot(object): if not whitelist: raise DependencyException('No currency pairs in whitelist') - + # fetching kline history for all pairs asynchronously and wait till all done - data = asyncio.get_event_loop().run_until_complete(self.exchange.async_get_tickers_history(whitelist, self.strategy.ticker_interval)) - + # data = asyncio.get_event_loop().run_until_complete(self.exchange.async_get_tickers_history(whitelist, self.strategy.ticker_interval)) + # list of pairs having buy signals buy_pairs = [] - # running get_signal on historical data fetched + # running get_signal on historical data fetched # to find buy signals - for _pair, thistory in data: + for _pair, thistory in self._klines.items(): (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) - if buy and not sell: + if buy and not sell: buy_pairs.append(_pair) - # If there is at least one buy signal then - # Go ahead and buy the first pair + # If there is at least one buy signal then + # Go ahead and buy the first pair if buy_pairs: return self.execute_buy(buy_pairs[0], stake_amount) - + return False def execute_buy(self, pair: str, stake_amount: float) -> bool: @@ -518,7 +522,8 @@ class FreqtradeBot(object): (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): - ticker = self.exchange.get_ticker_history(trade.pair, self.strategy.ticker_interval) + # ticker = self.exchange.get_ticker_history(trade.pair, self.strategy.ticker_interval) + ticker = self._klines[trade.pair] (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, ticker) From 31870abd251f1cf19fd0acf8890fb69443445006 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jul 2018 20:43:32 +0200 Subject: [PATCH 003/175] Refactor async-refresh to it's own function --- freqtrade/exchange/__init__.py | 28 +++++++++++++++++----------- freqtrade/freqtradebot.py | 21 ++++----------------- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index abbc9808b..8c5793b83 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -2,7 +2,7 @@ """ Cryptocurrency Exchanges support """ import logging from random import randint -from typing import List, Dict, Any, Optional +from typing import List, Dict, Tuple, Any, Optional from datetime import datetime from math import floor, ceil @@ -95,7 +95,7 @@ class Exchange(object): 'secret': exchange_config.get('secret'), 'password': exchange_config.get('password'), 'uid': exchange_config.get('uid', ''), - #'enableRateLimit': True, + # 'enableRateLimit': True, 'enableRateLimit': False, }) except (KeyError, AttributeError): @@ -334,23 +334,23 @@ class Exchange(object): logger.info("returning cached ticker-data for %s", pair) return self._cached_ticker[pair] - async def async_get_tickers_history(self, pairs, tick_interval): - # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ? - #loop = asyncio.new_event_loop() - #asyncio.set_event_loop(loop) - input_coroutines = [self.async_get_ticker_history(symbol, tick_interval) for symbol in pairs] + # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ? + # loop = asyncio.new_event_loop() + # asyncio.set_event_loop(loop) + input_coroutines = [self.async_get_ticker_history( + symbol, tick_interval) for symbol in pairs] tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) - #await self._api_async.close() + # await self._api_async.close() return tickers async def async_get_ticker_history(self, pair: str, tick_interval: str, - since_ms: Optional[int] = None) -> List[Dict]: + since_ms: Optional[int] = None) -> Tuple[str, List]: try: # fetch ohlcv asynchronously - print("fetching %s ..." % pair) + logger.debug("fetching %s ...", pair) data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) - print("done fetching %s ..." % pair) + logger.debug("done fetching %s ...", pair) return pair, data except ccxt.NotSupported as e: @@ -363,6 +363,12 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(f'Could not fetch ticker data. Msg: {e}') + def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> Dict: + logger.debug("Refreshing klines for %d pairs", len(pair_list)) + datatups = asyncio.get_event_loop().run_until_complete( + self.async_get_tickers_history(pair_list, ticker_interval)) + return {pair: data for (pair, data) in datatups} + @retrier def get_ticker_history(self, pair: str, tick_interval: str, since_ms: Optional[int] = None) -> List[Dict]: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e17498b51..eaa2184ac 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -8,7 +8,6 @@ import time import traceback from datetime import datetime from typing import Any, Callable, Dict, List, Optional -import asyncio import arrow import requests @@ -149,9 +148,7 @@ class FreqtradeBot(object): final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list self.config['exchange']['pair_whitelist'] = final_list - datatups = asyncio.get_event_loop().run_until_complete( - self.exchange.async_get_tickers_history(final_list, self.strategy.ticker_interval)) - self._klines = {pair: data for (pair, data) in datatups} + self._klines = self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval) # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -306,13 +303,7 @@ class FreqtradeBot(object): amount_reserve_percent += self.strategy.stoploss # it should not be more than 50% amount_reserve_percent = max(amount_reserve_percent, 0.5) - return min(min_stake_amounts)/amount_reserve_percent - - async def async_get_tickers(self, exchange, pairs): - input_coroutines = [exchange.async_get_ticker_history(symbol, self.strategy.ticker_interval) for symbol in pairs] - tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) - return tickers - #await exchange.close() + return min(min_stake_amounts) / amount_reserve_percent def create_trade(self) -> bool: """ @@ -341,17 +332,13 @@ class FreqtradeBot(object): if not whitelist: raise DependencyException('No currency pairs in whitelist') - - # fetching kline history for all pairs asynchronously and wait till all done - # data = asyncio.get_event_loop().run_until_complete(self.exchange.async_get_tickers_history(whitelist, self.strategy.ticker_interval)) - # list of pairs having buy signals buy_pairs = [] # running get_signal on historical data fetched # to find buy signals - for _pair, thistory in self._klines.items(): - (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) + for _pair in whitelist: + (buy, sell) = self.strategy.get_signal(_pair, interval, self._klines[_pair]) if buy and not sell: buy_pairs.append(_pair) From b45d465ed833264aaf1832b8f048fbbd4c6fc306 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jul 2018 20:50:59 +0200 Subject: [PATCH 004/175] init _klines properly --- freqtrade/freqtradebot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index eaa2184ac..4313b310d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -55,6 +55,7 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self._init_modules() + self._klines = {} def _init_modules(self) -> None: """ @@ -338,7 +339,7 @@ class FreqtradeBot(object): # running get_signal on historical data fetched # to find buy signals for _pair in whitelist: - (buy, sell) = self.strategy.get_signal(_pair, interval, self._klines[_pair]) + (buy, sell) = self.strategy.get_signal(_pair, interval, self._klines.get(_pair)) if buy and not sell: buy_pairs.append(_pair) From 52065178e16ad2e84a1774e73e6de5b878f3dbe1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jul 2018 20:53:32 +0200 Subject: [PATCH 005/175] use .get all the time --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4313b310d..365d145ca 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -511,7 +511,7 @@ class FreqtradeBot(object): experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): # ticker = self.exchange.get_ticker_history(trade.pair, self.strategy.ticker_interval) - ticker = self._klines[trade.pair] + ticker = self._klines.get(trade.pair) (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, ticker) From 12417cc303715244dc1701d13b5d5ed77d12b6ed Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jul 2018 20:54:51 +0200 Subject: [PATCH 006/175] fix tests --- freqtrade/tests/test_freqtradebot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 69f349107..fc84464e4 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -44,6 +44,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: """ freqtrade.strategy.get_signal = lambda e, s, t: value freqtrade.exchange.get_ticker_history = lambda p, i: None + freqtrade.exchange.refresh_tickers = lambda pl, i: {} def patch_RPCManager(mocker) -> MagicMock: From 136442245c850c83bb314230306a27ef1924992d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jul 2018 21:01:44 +0200 Subject: [PATCH 007/175] Add todo's and dockstring --- freqtrade/exchange/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 8c5793b83..8e88a7f5f 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -364,6 +364,12 @@ class Exchange(object): raise OperationalException(f'Could not fetch ticker data. Msg: {e}') def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> Dict: + """ + Refresh tickers asyncronously and return the result. + """ + # TODO: maybe add since_ms to use async in the download-script? + # TODO: only refresh once per interval ? *may require this to move to freqtradebot.py + # TODO@ Add tests for this and the async stuff above logger.debug("Refreshing klines for %d pairs", len(pair_list)) datatups = asyncio.get_event_loop().run_until_complete( self.async_get_tickers_history(pair_list, ticker_interval)) From c466a028e00341965acff78143f203a7a2abb13e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Aug 2018 21:19:49 +0200 Subject: [PATCH 008/175] Add a first async test --- .travis.yml | 4 +- freqtrade/tests/exchange/test_exchange.py | 53 +++++++++++++++++++++++ requirements.txt | 1 + 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 88121945f..981eedcf8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,12 +13,12 @@ addons: install: - ./install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH -- pip install --upgrade flake8 coveralls pytest-random-order mypy +- pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy - pip install -r requirements.txt - pip install -e . jobs: include: - - script: + - script: - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - coveralls - script: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index d327b97c7..b2c3b6a2e 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -27,6 +27,20 @@ def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, * assert api_mock.__dict__[mock_ccxt_fun].call_count == 1 +async def async_ccxt_exception(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): + with pytest.raises(TemporaryError): + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + await getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[mock_ccxt_fun].call_count == 1 + + with pytest.raises(OperationalException): + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + await getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[mock_ccxt_fun].call_count == 1 + + def test_init(default_conf, mocker, caplog): caplog.set_level(logging.INFO) get_patched_exchange(mocker, default_conf) @@ -515,6 +529,45 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=True) +@pytest.mark.asyncio +async def test_async_get_ticker_history(default_conf, mocker): + tick = [ + [ + 1511686200000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + async def async_fetch_ohlcv(pair, timeframe, since): + return tick + + exchange = get_patched_exchange(mocker, default_conf) + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = async_fetch_ohlcv + + exchange = Exchange(default_conf) + pair = 'ETH/BTC' + res = await exchange.async_get_ticker_history(pair, "5m") + assert type(res) is tuple + assert len(res) == 2 + assert res[0] == pair + assert res[1] == tick + + await async_ccxt_exception(mocker, default_conf, MagicMock(), + "async_get_ticker_history", "fetch_ohlcv", + pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + + api_mock = MagicMock() + with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'): + api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + await exchange.async_get_ticker_history(pair, "5m") + + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): if since: diff --git a/requirements.txt b/requirements.txt index 964da51e3..57dcf78e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ TA-Lib==0.4.17 pytest==3.6.4 pytest-mock==1.10.0 pytest-cov==2.5.1 +pytest-asyncio==0.9.0 tabulate==0.8.2 coinmarketcap==5.0.3 From 915160f21f88730ebcd0b09571e8c978078c9053 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Aug 2018 21:44:02 +0200 Subject: [PATCH 009/175] Add tests for tickers-history --- freqtrade/exchange/__init__.py | 2 +- freqtrade/tests/exchange/test_exchange.py | 47 +++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 8e88a7f5f..522636d22 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -334,7 +334,7 @@ class Exchange(object): logger.info("returning cached ticker-data for %s", pair) return self._cached_ticker[pair] - async def async_get_tickers_history(self, pairs, tick_interval): + async def async_get_tickers_history(self, pairs, tick_interval) -> List[Tuple[str, List]]: # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ? # loop = asyncio.new_event_loop() # asyncio.set_event_loop(loop) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index b2c3b6a2e..cf62a48c8 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -568,6 +568,53 @@ async def test_async_get_ticker_history(default_conf, mocker): await exchange.async_get_ticker_history(pair, "5m") +@pytest.mark.asyncio +async def test_async_get_tickers_history(default_conf, mocker): + tick = [ + [ + 1511686200000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + async def async_fetch_ohlcv(pair, timeframe, since): + return tick + + exchange = get_patched_exchange(mocker, default_conf) + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = async_fetch_ohlcv + + exchange = Exchange(default_conf) + pairs = ['ETH/BTC', 'XRP/BTC'] + res = await exchange.async_get_tickers_history(pairs, "5m") + assert type(res) is list + assert len(res) == 2 + assert type(res[0]) is tuple + assert res[0][0] == pairs[0] + assert res[0][1] == tick + assert res[1][0] == pairs[1] + assert res[1][1] == tick + + # await async_ccxt_exception(mocker, default_conf, MagicMock(), + # "async_get_tickers_history", "fetch_ohlcv", + # pairs=pairs, tick_interval=default_conf['ticker_interval']) + + # api_mock = MagicMock() + # with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'): + # api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) + # exchange = get_patched_exchange(mocker, default_conf, api_mock) + # await exchange.async_get_tickers_history('ETH/BTC', "5m") + + +def test_refresh_tickers(): + # TODO: Implement test for this + pass + + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): if since: From 9c08cdc81dec53be85d1d58f41ec56008e46f319 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Aug 2018 21:58:32 +0200 Subject: [PATCH 010/175] Fix typehints --- freqtrade/freqtradebot.py | 2 +- freqtrade/strategy/interface.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 365d145ca..f53bc2836 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -55,7 +55,7 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self._init_modules() - self._klines = {} + self._klines: Dict[str, List[Dict]] = {} def _init_modules(self) -> None: """ diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index dfd624393..488281eee 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -6,7 +6,7 @@ import logging from abc import ABC, abstractmethod from datetime import datetime from enum import Enum -from typing import Dict, List, NamedTuple, Tuple +from typing import Dict, List, NamedTuple, Optional, Tuple import warnings import arrow @@ -118,7 +118,7 @@ class IStrategy(ABC): dataframe = self.advise_sell(dataframe, metadata) return dataframe - def get_signal(self, pair: str, interval: str, ticker_hist: List[Dict]) -> Tuple[bool, bool]: + def get_signal(self, pair: str, interval: str, ticker_hist: Optional[List[Dict]]) -> Tuple[bool, bool]: """ Calculates current signal based several technical analysis indicators :param pair: pair in format ANT/BTC From 05ca78d2a3b9473b43836d3aab8d7f3335a27537 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 2 Aug 2018 17:10:38 +0200 Subject: [PATCH 011/175] ticker_history changed to candle_history naming --- freqtrade/tests/exchange/test_exchange.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 7cbead128..3a7573f70 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -530,7 +530,7 @@ def test_get_ticker(default_conf, mocker): @pytest.mark.asyncio -async def test_async_get_ticker_history(default_conf, mocker): +async def test_async_get_candle_history(default_conf, mocker): tick = [ [ 1511686200000, # unix timestamp ms @@ -551,25 +551,25 @@ async def test_async_get_ticker_history(default_conf, mocker): exchange = Exchange(default_conf) pair = 'ETH/BTC' - res = await exchange.async_get_ticker_history(pair, "5m") + res = await exchange.async_get_candle_history(pair, "5m") assert type(res) is tuple assert len(res) == 2 assert res[0] == pair assert res[1] == tick await async_ccxt_exception(mocker, default_conf, MagicMock(), - "async_get_ticker_history", "fetch_ohlcv", + "async_get_candle_history", "fetch_ohlcv", pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) api_mock = MagicMock() with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - await exchange.async_get_ticker_history(pair, "5m") + await exchange.async_get_candle_history(pair, "5m") @pytest.mark.asyncio -async def test_async_get_tickers_history(default_conf, mocker): +async def test_async_get_candles_history(default_conf, mocker): tick = [ [ 1511686200000, # unix timestamp ms @@ -590,7 +590,7 @@ async def test_async_get_tickers_history(default_conf, mocker): exchange = Exchange(default_conf) pairs = ['ETH/BTC', 'XRP/BTC'] - res = await exchange.async_get_tickers_history(pairs, "5m") + res = await exchange.async_get_candles_history(pairs, "5m") assert type(res) is list assert len(res) == 2 assert type(res[0]) is tuple @@ -600,14 +600,14 @@ async def test_async_get_tickers_history(default_conf, mocker): assert res[1][1] == tick # await async_ccxt_exception(mocker, default_conf, MagicMock(), - # "async_get_tickers_history", "fetch_ohlcv", + # "async_get_candles_history", "fetch_ohlcv", # pairs=pairs, tick_interval=default_conf['ticker_interval']) # api_mock = MagicMock() # with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'): # api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) # exchange = get_patched_exchange(mocker, default_conf, api_mock) - # await exchange.async_get_tickers_history('ETH/BTC', "5m") + # await exchange.async_get_candles_history('ETH/BTC', "5m") def test_refresh_tickers(): From 337d9174d93aac86ec1e33110d020996fd29dff9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Aug 2018 20:11:27 +0200 Subject: [PATCH 012/175] Flake8 fixes --- freqtrade/strategy/interface.py | 3 ++- freqtrade/tests/test_freqtradebot.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 488281eee..6d7b8dba7 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -118,7 +118,8 @@ class IStrategy(ABC): dataframe = self.advise_sell(dataframe, metadata) return dataframe - def get_signal(self, pair: str, interval: str, ticker_hist: Optional[List[Dict]]) -> Tuple[bool, bool]: + def get_signal(self, pair: str, interval: str, + ticker_hist: Optional[List[Dict]]) -> Tuple[bool, bool]: """ Calculates current signal based several technical analysis indicators :param pair: pair in format ANT/BTC diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index f3cb2be3f..3b72c3521 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -47,7 +47,6 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: freqtrade.exchange.refresh_tickers = lambda pl, i: {} - def patch_RPCManager(mocker) -> MagicMock: """ This function mock RPC manager to avoid repeating this code in almost every tests From c38d94df2d226ede89ab71f88dac5169fcd5b628 Mon Sep 17 00:00:00 2001 From: creslin Date: Fri, 3 Aug 2018 07:33:34 +0000 Subject: [PATCH 013/175] Resubmitting - because GIT. This is the last cut that was in #1117 before i closed that PR This PR allows a user to set the flag "ta_on_candle" in their config.json This will change the behaviour of the the bot to only process indicators when there is a new candle to be processed for that pair. The test is made up of "last dataframe row date + pair" is different to last_seen OR ta_on_candle is not True --- docs/configuration.md | 1 + freqtrade/constants.py | 1 + freqtrade/strategy/interface.py | 45 ++++++++++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index ff5ce118c..6f03117ba 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -23,6 +23,7 @@ The table below will list all configuration parameters. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes | `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. +| `ta_on_candle` | false | No | if set to true indicators are processed each new candle. If false each bot loop, this will mean the same candle is processed many times creating system load and buy/sell signals. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 87e354455..189b78617 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -53,6 +53,7 @@ CONF_SCHEMA = { }, 'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT}, 'dry_run': {'type': 'boolean'}, + 'ta_on_candle': {'type': 'boolean'}, 'minimal_roi': { 'type': 'object', 'patternProperties': { diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index dfd624393..e68d484de 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -19,6 +19,24 @@ from freqtrade.persistence import Trade logger = logging.getLogger(__name__) +class CandleAnalyzed: + ''' + Maintains candle_row, the last df ['date'], set by analyze_ticker. + This allows analyze_ticker to test if analysed the candle row in dataframe prior. + To not keep testing the same candle data, which is wasteful in CPU and time + ''' + def __init__(self, candle_row=0): + self.candle_row = candle_row + + def get_candle_row(self): + return self._candle_row + + def set_candle_row(self, row): + self._candle_row = row + + candle_row = property(get_candle_row, set_candle_row) + + class SignalType(Enum): """ Enum to distinguish between buy and sell signals @@ -72,6 +90,7 @@ class IStrategy(ABC): def __init__(self, config: dict) -> None: self.config = config + self.r = CandleAnalyzed() @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -112,10 +131,30 @@ class IStrategy(ABC): add several TA indicators and buy signal to it :return DataFrame with ticker data and indicator data """ + + # Test if seen this pair and last candle before. dataframe = parse_ticker_dataframe(ticker_history) - dataframe = self.advise_indicators(dataframe, metadata) - dataframe = self.advise_buy(dataframe, metadata) - dataframe = self.advise_sell(dataframe, metadata) + + last_seen = metadata['pair'] + str(dataframe.iloc[-1]['date']) + last_candle_processed = self.r.get_candle_row() + + if last_candle_processed != last_seen or self.config.get('ta_on_candle') is False: + # Defs that only make change on new candle data. + logging.info("TA Analysis Launched") + dataframe = self.advise_indicators(dataframe, metadata) + dataframe = self.advise_buy(dataframe, metadata) + dataframe = self.advise_sell(dataframe, metadata) + + last_seen = metadata['pair'] + str(dataframe.iloc[-1]['date']) + self.r.set_candle_row(last_seen) + else: + dataframe.loc['buy'] = 0 + dataframe.loc['sell'] = 0 + + # Other Defs in strategy that want to be called every loop here + # twitter_sell = self.watch_twitter_feed(dataframe, metadata) + logging.info("Loop Analysis Launched") + return dataframe def get_signal(self, pair: str, interval: str, ticker_hist: List[Dict]) -> Tuple[bool, bool]: From 6b3e8dcc3396753e649e9ecc7e9035810fced686 Mon Sep 17 00:00:00 2001 From: creslin Date: Fri, 3 Aug 2018 08:33:37 +0000 Subject: [PATCH 014/175] holds a dict of each pair last seen. to correctly manage the last seen of a pair. --- freqtrade/strategy/interface.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index e68d484de..b44d6997a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -21,20 +21,20 @@ logger = logging.getLogger(__name__) class CandleAnalyzed: ''' - Maintains candle_row, the last df ['date'], set by analyze_ticker. + Maintains dictionary of the last candle date a pair was processed with This allows analyze_ticker to test if analysed the candle row in dataframe prior. To not keep testing the same candle data, which is wasteful in CPU and time ''' - def __init__(self, candle_row=0): - self.candle_row = candle_row + def __init__(self, last_seen = {}): + self.last_seen = last_seen - def get_candle_row(self): - return self._candle_row + def get_last_seen(self, pair): + return self.last_seen.get(pair) - def set_candle_row(self, row): - self._candle_row = row + def set_last_seen(self, pair, candle_date): + self.last_seen[pair] = candle_date - candle_row = property(get_candle_row, set_candle_row) + candle_row = property(get_last_seen, set_last_seen) class SignalType(Enum): @@ -90,7 +90,7 @@ class IStrategy(ABC): def __init__(self, config: dict) -> None: self.config = config - self.r = CandleAnalyzed() + self.candleSeen = CandleAnalyzed() @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -135,18 +135,16 @@ class IStrategy(ABC): # Test if seen this pair and last candle before. dataframe = parse_ticker_dataframe(ticker_history) - last_seen = metadata['pair'] + str(dataframe.iloc[-1]['date']) - last_candle_processed = self.r.get_candle_row() + pair = metadata['pair'] + last_candle_seen = self.candleSeen.get_last_seen(pair) - if last_candle_processed != last_seen or self.config.get('ta_on_candle') is False: + if last_candle_seen != dataframe.iloc[-1]['date'] or self.config.get('ta_on_candle') is False: # Defs that only make change on new candle data. logging.info("TA Analysis Launched") dataframe = self.advise_indicators(dataframe, metadata) dataframe = self.advise_buy(dataframe, metadata) dataframe = self.advise_sell(dataframe, metadata) - - last_seen = metadata['pair'] + str(dataframe.iloc[-1]['date']) - self.r.set_candle_row(last_seen) + self.candleSeen.set_last_seen(pair=pair, candle_date=dataframe.iloc[-1]['date']) else: dataframe.loc['buy'] = 0 dataframe.loc['sell'] = 0 From d2a728cebd381bb7e5c1d7de218ab5ae4f31d418 Mon Sep 17 00:00:00 2001 From: creslin Date: Fri, 3 Aug 2018 08:38:13 +0000 Subject: [PATCH 015/175] flake 8 --- freqtrade/strategy/interface.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index b44d6997a..6035f50bf 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -25,7 +25,7 @@ class CandleAnalyzed: This allows analyze_ticker to test if analysed the candle row in dataframe prior. To not keep testing the same candle data, which is wasteful in CPU and time ''' - def __init__(self, last_seen = {}): + def __init__(self, last_seen={}): self.last_seen = last_seen def get_last_seen(self, pair): @@ -136,9 +136,9 @@ class IStrategy(ABC): dataframe = parse_ticker_dataframe(ticker_history) pair = metadata['pair'] - last_candle_seen = self.candleSeen.get_last_seen(pair) + last_seen = self.candleSeen.get_last_seen(pair) - if last_candle_seen != dataframe.iloc[-1]['date'] or self.config.get('ta_on_candle') is False: + if last_seen != dataframe.iloc[-1]['date'] or self.config.get('ta_on_candle') is False: # Defs that only make change on new candle data. logging.info("TA Analysis Launched") dataframe = self.advise_indicators(dataframe, metadata) From 1fef384bbae17b18e7d4842d47581a1cef7dae4f Mon Sep 17 00:00:00 2001 From: creslin Date: Fri, 3 Aug 2018 08:40:16 +0000 Subject: [PATCH 016/175] flake 8 --- freqtrade/strategy/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 6035f50bf..a17fdb625 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -135,7 +135,7 @@ class IStrategy(ABC): # Test if seen this pair and last candle before. dataframe = parse_ticker_dataframe(ticker_history) - pair = metadata['pair'] + pair = metadata.get('pair') last_seen = self.candleSeen.get_last_seen(pair) if last_seen != dataframe.iloc[-1]['date'] or self.config.get('ta_on_candle') is False: From 71b0e15182cf4bd45f816df0d3c1196f066cc2d9 Mon Sep 17 00:00:00 2001 From: creslin Date: Fri, 3 Aug 2018 08:45:24 +0000 Subject: [PATCH 017/175] updated configuration.md --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6f03117ba..4c43975d7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -23,7 +23,7 @@ The table below will list all configuration parameters. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes | `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. -| `ta_on_candle` | false | No | if set to true indicators are processed each new candle. If false each bot loop, this will mean the same candle is processed many times creating system load and buy/sell signals. +| `ta_on_candle` | false | No | if set to true indicators are processed each new candle. If false each bot loop, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). From 10ab6c7ffa54442c2d90e0b95653adb0be776bdc Mon Sep 17 00:00:00 2001 From: creslin Date: Fri, 3 Aug 2018 09:14:16 +0000 Subject: [PATCH 018/175] Removed unneeded property code --- freqtrade/strategy/interface.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index a17fdb625..5e0837767 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -34,8 +34,6 @@ class CandleAnalyzed: def set_last_seen(self, pair, candle_date): self.last_seen[pair] = candle_date - candle_row = property(get_last_seen, set_last_seen) - class SignalType(Enum): """ From 59b9a6d94d57db1addfe11c07ceb116401cda9db Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 3 Aug 2018 14:49:55 +0200 Subject: [PATCH 019/175] Break the loop as soon as one buy signal is found. --- freqtrade/freqtradebot.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4df3a327a..872f6dc5c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -333,20 +333,12 @@ class FreqtradeBot(object): if not whitelist: raise DependencyException('No currency pairs in whitelist') - # list of pairs having buy signals - buy_pairs = [] - # running get_signal on historical data fetched # to find buy signals for _pair in whitelist: (buy, sell) = self.strategy.get_signal(_pair, interval, self._klines.get(_pair)) if buy and not sell: - buy_pairs.append(_pair) - - # If there is at least one buy signal then - # Go ahead and buy the first pair - if buy_pairs: - return self.execute_buy(buy_pairs[0], stake_amount) + return self.execute_buy(_pair, stake_amount) return False From af93b18475f0eaaaa25996fc8ee92ca7e20c0956 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 3 Aug 2018 18:10:03 +0200 Subject: [PATCH 020/175] Do not refresh candles on "process_throttle_secs" but on intervals --- freqtrade/exchange/__init__.py | 15 ++------------- freqtrade/freqtradebot.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 897cb547a..46a56860b 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -6,10 +6,11 @@ from typing import List, Dict, Tuple, Any, Optional from datetime import datetime from math import floor, ceil +import asyncio import ccxt import ccxt.async_support as ccxt_async import arrow -import asyncio + from freqtrade import constants, OperationalException, DependencyException, TemporaryError @@ -362,18 +363,6 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(f'Could not fetch ticker data. Msg: {e}') - def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> Dict: - """ - Refresh tickers asyncronously and return the result. - """ - # TODO: maybe add since_ms to use async in the download-script? - # TODO: only refresh once per interval ? *may require this to move to freqtradebot.py - # TODO: Add tests for this and the async stuff above - logger.debug("Refreshing klines for %d pairs", len(pair_list)) - datatups = asyncio.get_event_loop().run_until_complete( - self.async_get_candles_history(pair_list, ticker_interval)) - return {pair: data for (pair, data) in datatups} - @retrier def get_candle_history(self, pair: str, tick_interval: str, since_ms: Optional[int] = None) -> List[Dict]: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 872f6dc5c..854b42c69 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -9,6 +9,7 @@ import traceback from datetime import datetime from typing import Any, Callable, Dict, List, Optional +import asyncio import arrow import requests @@ -56,6 +57,7 @@ class FreqtradeBot(object): self.exchange = Exchange(self.config) self._init_modules() self._klines: Dict[str, List[Dict]] = {} + self._klines_last_fetched_time = 0 def _init_modules(self) -> None: """ @@ -129,6 +131,34 @@ class FreqtradeBot(object): time.sleep(duration) return result + def refresh_tickers(self, pair_list: List[str]) -> Dict: + """ + Refresh tickers asyncronously and return the result. + """ + # TODO: maybe add since_ms to use async in the download-script? + # TODO: only refresh once per interval ? *may require this to move to freqtradebot.py + # TODO: Add tests for this and the async stuff above + + ticker_interval = self.strategy.ticker_interval + interval_in_seconds = int(ticker_interval[:-1]) * 60 + + should_not_update = ((self._klines_last_fetched_time + interval_in_seconds +1) > round(time.time())) + + if should_not_update: + return False + + logger.debug("Refreshing klines for %d pairs", len(pair_list)) + datatups = asyncio.get_event_loop().run_until_complete( + self.exchange.async_get_candles_history(pair_list, ticker_interval)) + + # fetching the timestamp of last candle + self._klines_last_fetched_time = datatups[0][1][-1][0] / 1000 + + # updating klines + self._klines = {pair: data for (pair, data) in datatups} + + return True + def _process(self, nb_assets: Optional[int] = 0) -> bool: """ Queries the persistence layer for open trades and handles them, @@ -149,7 +179,8 @@ class FreqtradeBot(object): final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list self.config['exchange']['pair_whitelist'] = final_list - self._klines = self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval) + # Refreshing candles + self.refresh_tickers(final_list) # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() From 3ce4d20ab9932b3695ce3bf8e4106c6f19430524 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 4 Aug 2018 13:04:16 +0200 Subject: [PATCH 021/175] using constants instead of stripping the string --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 854b42c69..c14411893 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -140,7 +140,7 @@ class FreqtradeBot(object): # TODO: Add tests for this and the async stuff above ticker_interval = self.strategy.ticker_interval - interval_in_seconds = int(ticker_interval[:-1]) * 60 + interval_in_seconds = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 should_not_update = ((self._klines_last_fetched_time + interval_in_seconds +1) > round(time.time())) From 4a9bf78770624357d91de4675a10c541fc488a74 Mon Sep 17 00:00:00 2001 From: Nullart2 Date: Sun, 5 Aug 2018 12:41:06 +0800 Subject: [PATCH 022/175] Order Book with tests --- config.json.example | 16 +++- config_full.json.example | 16 +++- docs/configuration.md | 7 ++ freqtrade/constants.py | 25 ++++- freqtrade/exchange/__init__.py | 23 +++++ freqtrade/exchange/exchange_helpers.py | 24 +++++ freqtrade/freqtradebot.py | 111 ++++++++++++++++++++-- freqtrade/tests/conftest.py | 16 ++++ freqtrade/tests/exchange/test_exchange.py | 10 ++ freqtrade/tests/test_freqtradebot.py | 39 +++++++- 10 files changed, 271 insertions(+), 16 deletions(-) diff --git a/config.json.example b/config.json.example index 8bd3942e6..c3dc6b5b6 100644 --- a/config.json.example +++ b/config.json.example @@ -37,7 +37,21 @@ "experimental": { "use_sell_signal": false, "sell_profit_only": false, - "ignore_roi_if_buy_signal": false + "ignore_roi_if_buy_signal": false, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + }, + "bid_strategy": { + "use_order_book": false, + "order_book_top": 2, + "percent_from_top": 0 + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9 + } }, "telegram": { "enabled": true, diff --git a/config_full.json.example b/config_full.json.example index cc3b3d630..5a364a93b 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -46,7 +46,21 @@ "experimental": { "use_sell_signal": false, "sell_profit_only": false, - "ignore_roi_if_buy_signal": false + "ignore_roi_if_buy_signal": false, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + }, + "bid_strategy": { + "use_order_book": false, + "order_book_top": 2, + "percent_from_top": 0 + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9 + }s }, "telegram": { "enabled": true, diff --git a/docs/configuration.md b/docs/configuration.md index ff5ce118c..5ffe24556 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,6 +39,13 @@ The table below will list all configuration parameters. | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. | `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` +| `experimental.check_depth_of_market` | false | No | Does not sell if the % difference of buy orders and sell orders is met in Order Book. +| `experimental.bids_to_ask_delta` | 0 | No | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. +| `experimental.bid_strategy.use_order_book` | false | No | Allows buying of pair using the rates in Order Book Bids. +| `experimental.bid_strategy.order_book_top` | 0 | No | Bot will use the top N rate in Order Book Bids. Ie. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids. +| `experimental.ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks. +| `experimental.ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. +| `experimental.ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `telegram.enabled` | true | Yes | Enable or not the usage of Telegram. | `telegram.token` | token | No | Your Telegram bot token. Only required if `telegram.enabled` is `true`. | `telegram.chat_id` | chat_id | No | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 87e354455..b7431af3c 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -89,7 +89,30 @@ CONF_SCHEMA = { 'properties': { 'use_sell_signal': {'type': 'boolean'}, 'sell_profit_only': {'type': 'boolean'}, - "ignore_roi_if_buy_signal_true": {'type': 'boolean'} + 'ignore_roi_if_buy_signal_true': {'type': 'boolean'}, + 'check_depth_of_market': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'bids_to_ask_delta': {'type': 'number', 'minimum': 0}, + } + }, + 'bid_strategy': { + 'type': 'object', + 'properties': { + 'percent_from_top': {'type': 'number', 'minimum': 0}, + 'use_order_book': {'type': 'boolean'}, + 'order_book_top': {'type': 'number', 'maximum': 20, 'minimum': 1} + } + }, + 'ask_strategy': { + 'type': 'object', + 'properties': { + 'use_order_book': {'type': 'boolean'}, + 'order_book_min': {'type': 'number', 'minimum': 1}, + 'order_book_max': {'type': 'number', 'minimum': 1, 'maximum': 50} + } + } } }, 'telegram': { diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 810957902..18b95b604 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -409,6 +409,29 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(e) + @retrier + def get_order_book(self, pair: str, limit: int = 100) -> dict: + try: + # 20180619: bittrex doesnt support limits -.- + # 20180619: binance support limits but only on specific range + if self._api.name == 'Binance': + limit_range = [5, 10, 20, 50, 100, 500, 1000] + for limitx in limit_range: + if limit <= limitx: + limit = limitx + break + + return self._api.fetch_l2_order_book(pair, limit) + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching order book.' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order book due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + @retrier def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: if self._conf['dry_run']: diff --git a/freqtrade/exchange/exchange_helpers.py b/freqtrade/exchange/exchange_helpers.py index 254c16309..6574f5a53 100644 --- a/freqtrade/exchange/exchange_helpers.py +++ b/freqtrade/exchange/exchange_helpers.py @@ -2,6 +2,7 @@ Functions to analyze ticker data with indicators and produce buy and sell signals """ import logging +import pandas as pd from pandas import DataFrame, to_datetime logger = logging.getLogger(__name__) @@ -31,3 +32,26 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame: }) frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle return frame + + +def order_book_to_dataframe(data: list) -> DataFrame: + """ + Gets order book list, returns dataframe with below format per suggested by creslin + ------------------------------------------------------------------- + b_sum b_size bids asks a_size a_sum + ------------------------------------------------------------------- + """ + cols = ['bids', 'b_size'] + bids_frame = DataFrame(data['bids'], columns=cols) + # add cumulative sum column + bids_frame['b_sum'] = bids_frame['b_size'].cumsum() + cols2 = ['asks', 'a_size'] + asks_frame = DataFrame(data['asks'], columns=cols2) + # add cumulative sum column + asks_frame['a_sum'] = asks_frame['a_size'].cumsum() + + frame = pd.concat([bids_frame['b_sum'], bids_frame['b_size'], bids_frame['bids'], + asks_frame['asks'], asks_frame['a_size'], asks_frame['a_sum']], axis=1, + keys=['b_sum', 'b_size', 'bids', 'asks', 'a_size', 'a_sum']) + # logger.info('order book %s', frame ) + return frame diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 46fbb3a38..0c4670971 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -21,6 +21,7 @@ from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from freqtrade.exchange.exchange_helpers import order_book_to_dataframe logger = logging.getLogger(__name__) @@ -233,16 +234,47 @@ class FreqtradeBot(object): return final_list - def get_target_bid(self, ticker: Dict[str, float]) -> float: + def get_target_bid(self, pair: str, ticker: Dict[str, float]) -> float: """ Calculates bid target between current ask price and last price :param ticker: Ticker to use for getting Ask and Last Price :return: float: Price """ if ticker['ask'] < ticker['last']: - return ticker['ask'] - balance = self.config['bid_strategy']['ask_last_balance'] - return ticker['ask'] + balance * (ticker['last'] - ticker['ask']) + ticker_rate = ticker['ask'] + else: + balance = self.config['bid_strategy']['ask_last_balance'] + ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask']) + + used_rate = ticker_rate + experimental_bid_strategy = self.config.get('experimental', {}).get('bid_strategy', {}) + if 'use_order_book' in experimental_bid_strategy and\ + experimental_bid_strategy.get('use_order_book', False): + logger.info('Getting price from order book') + order_book_top = experimental_bid_strategy.get('order_book_top', 1) + order_book = self.exchange.get_order_book(pair, order_book_top) + # top 1 = index 0 + order_book_rate = order_book['bids'][order_book_top - 1][0] + # if ticker has lower rate, then use ticker ( usefull if down trending ) + logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate) + if ticker_rate < order_book_rate: + logger.info('...using ticker rate instead %0.8f', ticker_rate) + used_rate = ticker_rate + used_rate = order_book_rate + else: + logger.info('Using Last Ask / Last Price') + used_rate = ticker_rate + percent_from_top = self.config.get('bid_strategy', {}).get('percent_from_top', 0) + if percent_from_top > 0: + used_rate = used_rate - (used_rate * percent_from_top) + used_rate = self._trunc_num(used_rate, 8) + logger.info('...percent_from_top enabled, new buy rate %0.8f', used_rate) + + return used_rate + + def _trunc_num(self, f, n): + import math + return math.floor(f * 10 ** n) / 10 ** n def _get_trade_stake_amount(self) -> Optional[float]: """ @@ -334,9 +366,37 @@ class FreqtradeBot(object): (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) if buy and not sell: + experimental_check_depth_of_market = self.config.get('experimental', {}).\ + get('check_depth_of_market', {}) + if (experimental_check_depth_of_market.get('enabled', False)) and\ + (experimental_check_depth_of_market.get('bids_to_ask_delta', 0) > 0): + if self._check_depth_of_market_buy(_pair): + return self.execute_buy(_pair, stake_amount) + else: + return False return self.execute_buy(_pair, stake_amount) return False + def _check_depth_of_market_buy(self, pair: str, ) -> bool: + """ + Checks depth of market before executing a buy + """ + experimental_check_depth_of_market = self.config.get('experimental', {}).\ + get('check_depth_of_market', {}) + conf_bids_to_ask_delta = experimental_check_depth_of_market.get('bids_to_ask_delta', 0) + logger.info('checking depth of market for %s', pair) + order_book = self.exchange.get_order_book(pair, 1000) + order_book_data_frame = order_book_to_dataframe(order_book) + order_book_bids = order_book_data_frame['b_size'].sum() + order_book_asks = order_book_data_frame['a_size'].sum() + bids_ask_delta = order_book_bids / order_book_asks + logger.info('bids: %s, asks: %s, delta: %s', order_book_bids, + order_book_asks, + order_book_bids / order_book_asks) + if bids_ask_delta >= conf_bids_to_ask_delta: + return True + return False + def execute_buy(self, pair: str, stake_amount: float) -> bool: """ Executes a limit buy for the given pair @@ -349,7 +409,7 @@ class FreqtradeBot(object): fiat_currency = self.config.get('fiat_display_currency', None) # Calculate amount - buy_limit = self.get_target_bid(self.exchange.get_ticker(pair)) + buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair)) min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit) if min_stake_amount is not None and min_stake_amount > stake_amount: @@ -492,7 +552,7 @@ class FreqtradeBot(object): raise ValueError(f'attempt to handle closed trade: {trade}') logger.debug('Handling %s ...', trade) - current_rate = self.exchange.get_ticker(trade.pair)['bid'] + sell_rate = self.exchange.get_ticker(trade.pair)['bid'] (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) @@ -501,13 +561,44 @@ class FreqtradeBot(object): (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, ticker) - should_sell = self.strategy.should_sell(trade, current_rate, datetime.utcnow(), buy, sell) - if should_sell.sell_flag: - self.execute_sell(trade, current_rate, should_sell.sell_type) - return True + experimental_ask_strategy = self.config.get('experimental', {}).get('ask_strategy', {}) + if 'use_order_book' in experimental_ask_strategy and\ + experimental_ask_strategy.get('use_order_book', False): + logger.info('Using order book for selling...') + # logger.debug('Order book %s',orderBook) + order_book_min = experimental_ask_strategy.get('order_book_min', 1) + order_book_max = experimental_ask_strategy.get('order_book_max', 1) + + order_book = self.exchange.get_order_book(trade.pair, order_book_max) + + for i in range(order_book_min, order_book_max + 1): + order_book_rate = order_book['asks'][i - 1][0] + + # if orderbook has higher rate (high profit), + # use orderbook, otherwise just use bids rate + logger.info(' order book asks top %s: %0.8f', i, order_book_rate) + if sell_rate < order_book_rate: + sell_rate = order_book_rate + + if self.check_sell(trade, sell_rate, buy, sell): + return True + break + else: + logger.info('checking sell') + if self.check_sell(trade, sell_rate, buy, sell): + return True + logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False + def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: + should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) + if should_sell.sell_flag: + self.execute_sell(trade, sell_rate, should_sell.sell_type) + logger.info('excuted sell') + return True + return False + def check_handle_timedout(self) -> None: """ Check if any orders are timed out and cancel if neccessary diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index d18016e16..d7f7e96d9 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -116,6 +116,22 @@ def default_conf(): "NEO/BTC" ] }, + "experimental": { + "check_depth_of_market": { + "enabled": False, + "bids_to_ask_delta": 1 + }, + "bid_strategy": { + "percent_from_top": 0, + "use_order_book": False, + "order_book_top": 1 + }, + "ask_strategy": { + "use_order_book": False, + "order_book_min": 1, + "order_book_max": 1 + } + }, "telegram": { "enabled": True, "token": "token", diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index d327b97c7..d0917bda2 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -515,6 +515,16 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=True) +def test_get_order_book(default_conf, mocker): + default_conf['exchange']['name'] = 'binance' + exchange = Exchange(default_conf) + order_book = exchange.get_order_book(pair='ETH/BTC', limit=50) + assert 'bids' in order_book + assert 'asks' in order_book + assert len(order_book['bids']) == 50 + assert len(order_book['asks']) == 50 + + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): if since: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 69f349107..4718ab0fa 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -664,21 +664,21 @@ def test_balance_fully_ask_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 0.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 20 + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) == 20 def test_balance_fully_last_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 1.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 10 + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) == 10 def test_balance_bigger_last_ask(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 1.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.get_target_bid({'ask': 5, 'last': 10}) == 5 + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 5, 'last': 10}) == 5 def test_process_maybe_execute_buy(mocker, default_conf) -> None: @@ -1876,3 +1876,36 @@ def test_get_real_amount_open_trade(default_conf, mocker): freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) assert freqtrade.get_real_amount(trade, order) == amount + + +def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, markets, mocker): + default_conf['experimental']['check_depth_of_market']['enabled'] = True + default_conf['experimental']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 + patch_RPCManager(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets + ) + + # Save state of current whitelist + whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade is not None + assert trade.stake_amount == 0.001 + assert trade.is_open + assert trade.open_date is not None + assert trade.exchange == 'bittrex' + + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) + + assert trade.open_rate == 0.00001099 + assert whitelist == default_conf['exchange']['pair_whitelist'] From 26d591ea43dfc8a9799ab80bbc0e7bdbdd51f724 Mon Sep 17 00:00:00 2001 From: Nullart2 Date: Sun, 5 Aug 2018 21:08:07 +0800 Subject: [PATCH 023/175] mypy fix --- freqtrade/exchange/__init__.py | 2 +- freqtrade/exchange/exchange_helpers.py | 7 ++++--- freqtrade/freqtradebot.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 18b95b604..d5be25ba0 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -116,7 +116,7 @@ class Exchange(object): api.urls['api'] = api.urls['test'] logger.info("Enabled Sandbox API on %s", name) else: - logger.warning(self, "No Sandbox URL in CCXT, exiting. " + logger.warning(name, "No Sandbox URL in CCXT, exiting. " "Please check your config.json") raise OperationalException(f'Exchange {name} does not provide a sandbox api') diff --git a/freqtrade/exchange/exchange_helpers.py b/freqtrade/exchange/exchange_helpers.py index 6574f5a53..ac729190a 100644 --- a/freqtrade/exchange/exchange_helpers.py +++ b/freqtrade/exchange/exchange_helpers.py @@ -34,7 +34,7 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame: return frame -def order_book_to_dataframe(data: list) -> DataFrame: +def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: """ Gets order book list, returns dataframe with below format per suggested by creslin ------------------------------------------------------------------- @@ -42,11 +42,12 @@ def order_book_to_dataframe(data: list) -> DataFrame: ------------------------------------------------------------------- """ cols = ['bids', 'b_size'] - bids_frame = DataFrame(data['bids'], columns=cols) + + bids_frame = DataFrame(bids, columns=cols) # add cumulative sum column bids_frame['b_sum'] = bids_frame['b_size'].cumsum() cols2 = ['asks', 'a_size'] - asks_frame = DataFrame(data['asks'], columns=cols2) + asks_frame = DataFrame(asks, columns=cols2) # add cumulative sum column asks_frame['a_sum'] = asks_frame['a_size'].cumsum() diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0c4670971..b91cb1bd2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -386,7 +386,7 @@ class FreqtradeBot(object): conf_bids_to_ask_delta = experimental_check_depth_of_market.get('bids_to_ask_delta', 0) logger.info('checking depth of market for %s', pair) order_book = self.exchange.get_order_book(pair, 1000) - order_book_data_frame = order_book_to_dataframe(order_book) + order_book_data_frame = order_book_to_dataframe(order_book['bids'], order_book['asks']) order_book_bids = order_book_data_frame['b_size'].sum() order_book_asks = order_book_data_frame['a_size'].sum() bids_ask_delta = order_book_bids / order_book_asks From 7143b64fb73411744686381770a3adbf1897da6c Mon Sep 17 00:00:00 2001 From: Nullart2 Date: Sun, 5 Aug 2018 22:41:58 +0800 Subject: [PATCH 024/175] tests for coverage --- freqtrade/freqtradebot.py | 7 +- freqtrade/tests/exchange/test_exchange.py | 16 +++++ freqtrade/tests/test_freqtradebot.py | 81 +++++++++++++++++++++++ 3 files changed, 98 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index b91cb1bd2..f1955a9d7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -264,11 +264,6 @@ class FreqtradeBot(object): else: logger.info('Using Last Ask / Last Price') used_rate = ticker_rate - percent_from_top = self.config.get('bid_strategy', {}).get('percent_from_top', 0) - if percent_from_top > 0: - used_rate = used_rate - (used_rate * percent_from_top) - used_rate = self._trunc_num(used_rate, 8) - logger.info('...percent_from_top enabled, new buy rate %0.8f', used_rate) return used_rate @@ -377,7 +372,7 @@ class FreqtradeBot(object): return self.execute_buy(_pair, stake_amount) return False - def _check_depth_of_market_buy(self, pair: str, ) -> bool: + def _check_depth_of_market_buy(self, pair: str) -> bool: """ Checks depth of market before executing a buy """ diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index d0917bda2..60f51553a 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -525,6 +525,22 @@ def test_get_order_book(default_conf, mocker): assert len(order_book['asks']) == 50 +def test_get_order_book_exception(default_conf, mocker): + api_mock = MagicMock() + with pytest.raises(OperationalException): + api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.NotSupported) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_order_book(pair='ETH/BTC', limit=50) + with pytest.raises(TemporaryError): + api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_order_book(pair='ETH/BTC', limit=50) + with pytest.raises(OperationalException): + api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_order_book(pair='ETH/BTC', limit=50) + + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): if since: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 4718ab0fa..563f9961b 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1909,3 +1909,84 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, assert trade.open_rate == 0.00001099 assert whitelist == default_conf['exchange']['pair_whitelist'] + + +def test_order_book_depth_of_market_high_delta(default_conf, ticker, + limit_buy_order, fee, markets, mocker): + default_conf['experimental']['check_depth_of_market']['enabled'] = True + default_conf['experimental']['check_depth_of_market']['bids_to_ask_delta'] = 100 + patch_RPCManager(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets + ) + + # Save state of current whitelist + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade is None + + +def test_order_book_bid_strategy(default_conf) -> None: + default_conf['exchange']['name'] = 'binance' + default_conf['experimental']['bid_strategy']['use_order_book'] = True + default_conf['experimental']['bid_strategy']['order_book_top'] = 2 + default_conf['telegram']['enabled'] = False + + freqtrade = FreqtradeBot(default_conf) + + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) != 20 + + +def test_trunc_num(default_conf) -> None: + default_conf['telegram']['enabled'] = False + freqtrade = FreqtradeBot(default_conf) + + assert freqtrade._trunc_num(10.1111, 2) == 10.11 + + +def test_check_depth_of_market_buy(default_conf) -> None: + default_conf['telegram']['enabled'] = False + default_conf['exchange']['name'] = 'binance' + default_conf['experimental']['check_depth_of_market']['enabled'] = True + default_conf['experimental']['check_depth_of_market']['bids_to_ask_delta'] = 100 + freqtrade = FreqtradeBot(default_conf) + + assert freqtrade._check_depth_of_market_buy('ETH/BTC') is False + + +def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order, + fee, markets, mocker) -> None: + default_conf['exchange']['name'] = 'binance' + default_conf['experimental']['ask_strategy']['use_order_book'] = True + default_conf['experimental']['ask_strategy']['order_book_min'] = 1 + default_conf['experimental']['ask_strategy']['order_book_max'] = 2 + default_conf['telegram']['enabled'] = False + patch_RPCManager(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + sell=MagicMock(return_value={'id': limit_sell_order['id']}), + get_fee=fee, + get_markets=markets + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade From 1309c2b14f50c55acdbdeaaf217aba34dfd7656b Mon Sep 17 00:00:00 2001 From: Nullart2 Date: Sun, 5 Aug 2018 22:56:14 +0800 Subject: [PATCH 025/175] tests update --- freqtrade/freqtradebot.py | 6 +++--- freqtrade/tests/test_freqtradebot.py | 10 +++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f1955a9d7..496ef1168 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -260,7 +260,8 @@ class FreqtradeBot(object): if ticker_rate < order_book_rate: logger.info('...using ticker rate instead %0.8f', ticker_rate) used_rate = ticker_rate - used_rate = order_book_rate + else: + used_rate = order_book_rate else: logger.info('Using Last Ask / Last Price') used_rate = ticker_rate @@ -557,8 +558,7 @@ class FreqtradeBot(object): ticker) experimental_ask_strategy = self.config.get('experimental', {}).get('ask_strategy', {}) - if 'use_order_book' in experimental_ask_strategy and\ - experimental_ask_strategy.get('use_order_book', False): + if experimental_ask_strategy.get('use_order_book', False): logger.info('Using order book for selling...') # logger.debug('Order book %s',orderBook) order_book_min = experimental_ask_strategy.get('order_book_min', 1) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 563f9961b..6282720d6 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1938,11 +1938,12 @@ def test_order_book_bid_strategy(default_conf) -> None: default_conf['exchange']['name'] = 'binance' default_conf['experimental']['bid_strategy']['use_order_book'] = True default_conf['experimental']['bid_strategy']['order_book_top'] = 2 + default_conf['bid_strategy']['ask_last_balance'] = 0 default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) - assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) != 20 + assert freqtrade.get_target_bid('BTC/USDT', {'ask': 2, 'last': 2}) == 2 def test_trunc_num(default_conf) -> None: @@ -1990,3 +1991,10 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order trade = Trade.query.first() assert trade + + time.sleep(0.01) # Race condition fix + trade.update(limit_buy_order) + assert trade.is_open is True + + patch_get_signal(freqtrade, value=(False, True)) + assert freqtrade.handle_trade(trade) is True From 255f30385031003bd2915809fa8c3bd9fc769323 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Aug 2018 08:56:06 +0200 Subject: [PATCH 026/175] Fix tests and flake8 --- freqtrade/freqtradebot.py | 8 ++++---- freqtrade/tests/test_freqtradebot.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c14411893..f4e9c1d5b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -131,18 +131,18 @@ class FreqtradeBot(object): time.sleep(duration) return result - def refresh_tickers(self, pair_list: List[str]) -> Dict: + def refresh_tickers(self, pair_list: List[str]) -> bool: """ Refresh tickers asyncronously and return the result. """ # TODO: maybe add since_ms to use async in the download-script? - # TODO: only refresh once per interval ? *may require this to move to freqtradebot.py # TODO: Add tests for this and the async stuff above - + ticker_interval = self.strategy.ticker_interval interval_in_seconds = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 - should_not_update = ((self._klines_last_fetched_time + interval_in_seconds +1) > round(time.time())) + should_not_update = ((self._klines_last_fetched_time + + interval_in_seconds + 1) > round(time.time())) if should_not_update: return False diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 3b72c3521..3bf6ad037 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -44,7 +44,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: """ freqtrade.strategy.get_signal = lambda e, s, t: value freqtrade.exchange.get_candle_history = lambda p, i: None - freqtrade.exchange.refresh_tickers = lambda pl, i: {} + freqtrade.refresh_tickers = lambda i: True def patch_RPCManager(mocker) -> MagicMock: From c9580b31d0b2e7cd4e80a7b91f12d6fbc76f8ef8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Aug 2018 09:25:21 +0200 Subject: [PATCH 027/175] parametrize outdated_offset to simplify sandbox usage --- config_full.json.example | 3 +- docs/sandbox-testing.md | 120 +++++++++++++++----------------- freqtrade/constants.py | 3 +- freqtrade/strategy/interface.py | 3 +- 4 files changed, 61 insertions(+), 68 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index cc3b3d630..a06eeb7e6 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -41,7 +41,8 @@ ], "pair_blacklist": [ "DOGE/BTC" - ] + ], + "outdated_offset": 5 }, "experimental": { "use_sell_signal": false, diff --git a/docs/sandbox-testing.md b/docs/sandbox-testing.md index 572fbccef..7f3457d15 100644 --- a/docs/sandbox-testing.md +++ b/docs/sandbox-testing.md @@ -1,4 +1,5 @@ # Sandbox API testing + Where an exchange provides a sandbox for risk-free integration, or end-to-end, testing CCXT provides access to these. This document is a *light overview of configuring Freqtrade and GDAX sandbox. @@ -11,8 +12,11 @@ https://public.sandbox.gdax.com https://api-public.sandbox.gdax.com --- + # Configure a Sandbox account on Gdax + Aim of this document section + - An sanbox account - create 2FA (needed to create an API) - Add test 50BTC to account @@ -30,122 +34,108 @@ After registration and Email confimation you wil be redirected into your sanbox > https://public.sandbox.pro.coinbase.com/ ## Enable 2Fa (a prerequisite to creating sandbox API Keys) + From within sand box site select your profile, top right. >Or as a direct link: https://public.sandbox.pro.coinbase.com/profile -From the menu panel to the left of the screen select +From the menu panel to the left of the screen select + > Security: "*View or Update*" -In the new site select "enable authenticator" as typical google Authenticator. -- open Google Authenticator on your phone -- scan barcode -- enter your generated 2fa +In the new site select "enable authenticator" as typical google Authenticator. + +- open Google Authenticator on your phone +- scan barcode +- enter your generated 2fa + +## Enable API Access -## Enable API Access From within sandbox select profile>api>create api-keys >or as a direct link: https://public.sandbox.pro.coinbase.com/profile/api -Click on "create one" and ensure **view** and **trade** are "checked" and sumbit your 2Fa +Click on "create one" and ensure **view** and **trade** are "checked" and sumbit your 2FA + - **Copy and paste the Passphase** into a notepade this will be needed later - **Copy and paste the API Secret** popup into a notepad this will needed later - **Copy and paste the API Key** into a notepad this will needed later ## Add 50 BTC test funds -To add funds, use the web interface deposit and withdraw buttons. +To add funds, use the web interface deposit and withdraw buttons. To begin select 'Wallets' from the top menu. > Or as a direct link: https://public.sandbox.pro.coinbase.com/wallets - Deposits (bottom left of screen) -- - Deposit Funds Bitcoin -- - - Coinbase BTC Wallet -- - - - Max (50 BTC) +- - Deposit Funds Bitcoin +- - - Coinbase BTC Wallet +- - - - Max (50 BTC) - - - - - Deposit *This process may be repeated for other currencies, ETH as example* + --- + # Configure Freqtrade to use Gax Sandbox The aim of this document section - - Enable sandbox URLs in Freqtrade - - Configure API - - - secret - - - key - - - passphrase + +- Enable sandbox URLs in Freqtrade +- Configure API +- - secret +- - key +- - passphrase ## Sandbox URLs -Freqtrade makes use of CCXT which in turn provides a list of URLs to Freqtrade. -These include `['test']` and `['api']`. + +Freqtrade makes use of CCXT which in turn provides a list of URLs to Freqtrade. +These include `['test']` and `['api']`. + - `[Test]` if available will point to an Exchanges sandbox. - `[Api]` normally used, and resolves to live API target on the exchange To make use of sandbox / test add "sandbox": true, to your config.json -``` + +```json "exchange": { "name": "gdax", "sandbox": true, "key": "5wowfxemogxeowo;heiohgmd", "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==", "password": "1bkjfkhfhfu6sr", + "outdated_offset": 5 "pair_whitelist": [ "BTC/USD" ``` + Also insert your + - api-key (noted earlier) - api-secret (noted earlier) - password (the passphrase - noted earlier) --- -## You should now be ready to test your sandbox! + +## You should now be ready to test your sandbox + Ensure Freqtrade logs show the sandbox URL, and trades made are shown in sandbox. -** Typically the BTC/USD has the most activity in sandbox to test against. +** Typically the BTC/USD has the most activity in sandbox to test against. ## GDAX - Old Candles problem -It is my experience that GDAX sandbox candles may be 20+- minutes out of date. This can cause trades to fail as one of Freqtrades safety checks -To disable this check, edit: ->strategy/interface.py -Look for the following section: -``` - # Check if dataframe is out of date - signal_date = arrow.get(latest['date']) - interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] - if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))): - logger.warning( - 'Outdated history for pair %s. Last tick is %s minutes old', - pair, - (arrow.utcnow() - signal_date).seconds // 60 - ) - return False, False -``` +It is my experience that GDAX sandbox candles may be 20+- minutes out of date. This can cause trades to fail as one of Freqtrades safety checks. -You could Hash out the entire check as follows: +To disable this check, add / change the `"outdated_offset"` parameter in the exchange section of your configuration to adjust for this delay. +Example based on the above configuration: + +```json + "exchange": { + "name": "gdax", + "sandbox": true, + "key": "5wowfxemogxeowo;heiohgmd", + "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==", + "password": "1bkjfkhfhfu6sr", + "outdated_offset": 30 + "pair_whitelist": [ + "BTC/USD" ``` - # # Check if dataframe is out of date - # signal_date = arrow.get(latest['date']) - # interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] - # if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))): - # logger.warning( - # 'Outdated history for pair %s. Last tick is %s minutes old', - # pair, - # (arrow.utcnow() - signal_date).seconds // 60 - # ) - # return False, False - ``` - - Or inrease the timeout to offer a level of protection/alignment of this test to freqtrade in live. - - As example, to allow an additional 30 minutes. "(interval_minutes * 2 + 5 + 30)" - ``` - # Check if dataframe is out of date - signal_date = arrow.get(latest['date']) - interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] - if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5 + 30))): - logger.warning( - 'Outdated history for pair %s. Last tick is %s minutes old', - pair, - (arrow.utcnow() - signal_date).seconds // 60 - ) - return False, False -``` \ No newline at end of file diff --git a/freqtrade/constants.py b/freqtrade/constants.py index b30add71b..175d09405 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -145,7 +145,8 @@ CONF_SCHEMA = { 'pattern': '^[0-9A-Z]+/[0-9A-Z]+$' }, 'uniqueItems': True - } + }, + 'outdated_offset': {'type': 'integer', 'minimum': 1} }, 'required': ['name', 'key', 'secret', 'pair_whitelist'] } diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index dfd624393..aa1e903de 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -155,7 +155,8 @@ class IStrategy(ABC): # Check if dataframe is out of date signal_date = arrow.get(latest['date']) interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] - if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))): + offset = self.config.get('outdated_offset', 5) + if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + offset))): logger.warning( 'Outdated history for pair %s. Last tick is %s minutes old', pair, From c9c0e108ab249df607699c4b8ea066517e664e24 Mon Sep 17 00:00:00 2001 From: Nullart2 Date: Tue, 7 Aug 2018 18:29:37 +0800 Subject: [PATCH 028/175] refactor --- freqtrade/exchange/__init__.py | 20 +++++++++---- freqtrade/freqtradebot.py | 15 +++------- freqtrade/tests/test_freqtradebot.py | 43 +++++++++++++++++++++------- 3 files changed, 51 insertions(+), 27 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index d5be25ba0..4b78283ea 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -411,15 +411,23 @@ class Exchange(object): @retrier def get_order_book(self, pair: str, limit: int = 100) -> dict: + """ + get order book level 2 from exchange + + Notes: + 20180619: bittrex doesnt support limits -.- + 20180619: binance support limits but only on specific range + """ try: - # 20180619: bittrex doesnt support limits -.- - # 20180619: binance support limits but only on specific range if self._api.name == 'Binance': limit_range = [5, 10, 20, 50, 100, 500, 1000] - for limitx in limit_range: - if limit <= limitx: - limit = limitx - break + # get next-higher step in the limit_range list + limit = min(list(filter(lambda x: limit <= x, limit_range))) + # above script works like loop below (but with slightly better performance): + # for limitx in limit_range: + # if limit <= limitx: + # limit = limitx + # break return self._api.fetch_l2_order_book(pair, limit) except ccxt.NotSupported as e: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 496ef1168..55b9c577f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -268,10 +268,6 @@ class FreqtradeBot(object): return used_rate - def _trunc_num(self, f, n): - import math - return math.floor(f * 10 ** n) / 10 ** n - def _get_trade_stake_amount(self) -> Optional[float]: """ Check if stake amount can be fulfilled with the available balance @@ -366,20 +362,18 @@ class FreqtradeBot(object): get('check_depth_of_market', {}) if (experimental_check_depth_of_market.get('enabled', False)) and\ (experimental_check_depth_of_market.get('bids_to_ask_delta', 0) > 0): - if self._check_depth_of_market_buy(_pair): + if self._check_depth_of_market_buy(_pair, experimental_check_depth_of_market): return self.execute_buy(_pair, stake_amount) else: return False return self.execute_buy(_pair, stake_amount) return False - def _check_depth_of_market_buy(self, pair: str) -> bool: + def _check_depth_of_market_buy(self, pair: str, conf: Dict) -> bool: """ Checks depth of market before executing a buy """ - experimental_check_depth_of_market = self.config.get('experimental', {}).\ - get('check_depth_of_market', {}) - conf_bids_to_ask_delta = experimental_check_depth_of_market.get('bids_to_ask_delta', 0) + conf_bids_to_ask_delta = conf.get('bids_to_ask_delta', 0) logger.info('checking depth of market for %s', pair) order_book = self.exchange.get_order_book(pair, 1000) order_book_data_frame = order_book_to_dataframe(order_book['bids'], order_book['asks']) @@ -387,8 +381,7 @@ class FreqtradeBot(object): order_book_asks = order_book_data_frame['a_size'].sum() bids_ask_delta = order_book_bids / order_book_asks logger.info('bids: %s, asks: %s, delta: %s', order_book_bids, - order_book_asks, - order_book_bids / order_book_asks) + order_book_asks, bids_ask_delta) if bids_ask_delta >= conf_bids_to_ask_delta: return True return False diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 6282720d6..4357b573a 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -159,6 +159,15 @@ def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: assert whitelist == [] +def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) + + with pytest.raises(OperationalException): + freqtrade._gen_pair_whitelist(base_currency='BTC') + + @pytest.mark.skip(reason="Test not implemented") def test_refresh_whitelist() -> None: pass @@ -1914,6 +1923,7 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_order, fee, markets, mocker): default_conf['experimental']['check_depth_of_market']['enabled'] = True + # delta is 100 which is impossible to reach. hence check_depth_of_market will return false default_conf['experimental']['check_depth_of_market']['bids_to_ask_delta'] = 100 patch_RPCManager(mocker) mocker.patch.multiple( @@ -1924,7 +1934,6 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, get_fee=fee, get_markets=markets ) - # Save state of current whitelist freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -1934,33 +1943,47 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, assert trade is None -def test_order_book_bid_strategy(default_conf) -> None: +def test_order_book_bid_strategy1(default_conf) -> None: + """ + test if function get_target_bid will return the order book price + instead of the ask rate + """ default_conf['exchange']['name'] = 'binance' default_conf['experimental']['bid_strategy']['use_order_book'] = True default_conf['experimental']['bid_strategy']['order_book_top'] = 2 default_conf['bid_strategy']['ask_last_balance'] = 0 default_conf['telegram']['enabled'] = False + freqtrade = FreqtradeBot(default_conf) + assert freqtrade.get_target_bid('BTC/USDT', {'ask': 200000, 'last': 200000}) != 200000 + + +def test_order_book_bid_strategy2(default_conf) -> None: + """ + test if function get_target_bid will return ask rate instead + of the order book rate + """ + default_conf['exchange']['name'] = 'binance' + default_conf['experimental']['bid_strategy']['use_order_book'] = True + default_conf['experimental']['bid_strategy']['order_book_top'] = 1 + default_conf['bid_strategy']['ask_last_balance'] = 0 + default_conf['telegram']['enabled'] = False + freqtrade = FreqtradeBot(default_conf) assert freqtrade.get_target_bid('BTC/USDT', {'ask': 2, 'last': 2}) == 2 -def test_trunc_num(default_conf) -> None: - default_conf['telegram']['enabled'] = False - freqtrade = FreqtradeBot(default_conf) - - assert freqtrade._trunc_num(10.1111, 2) == 10.11 - - def test_check_depth_of_market_buy(default_conf) -> None: default_conf['telegram']['enabled'] = False default_conf['exchange']['name'] = 'binance' default_conf['experimental']['check_depth_of_market']['enabled'] = True + # delta is 100 which is impossible to reach. hence function will return false default_conf['experimental']['check_depth_of_market']['bids_to_ask_delta'] = 100 freqtrade = FreqtradeBot(default_conf) - assert freqtrade._check_depth_of_market_buy('ETH/BTC') is False + conf = default_conf['experimental']['check_depth_of_market'] + assert freqtrade._check_depth_of_market_buy('ETH/BTC', conf) is False def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order, From 3c451e067716f65c656729dcc4a95333836d0c55 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Aug 2018 21:54:52 +0200 Subject: [PATCH 029/175] Add test for bugreport #1111 --- freqtrade/tests/rpc/test_rpc.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 70b7dcfd9..01130fccf 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -6,6 +6,7 @@ from unittest.mock import MagicMock, ANY import pytest +from freqtrade import TemporaryError from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade @@ -320,6 +321,32 @@ def test_rpc_balance_handle(default_conf, mocker): 'pending': 2.0, 'est_btc': 12.0, }] + assert result['total'] == 12.0 + + mock_balance = { + 'ETH': { + 'free': 10.0, + 'total': 12.0, + 'used': 2.0, + } + } + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_balances=MagicMock(return_value=mock_balance), + get_ticker=MagicMock(side_effect=TemporaryError('Could not load ticker due to xxx')) + ) + result = rpc._rpc_balance(default_conf['fiat_display_currency']) + assert prec_satoshi(result['total'], 12) + assert prec_satoshi(result['value'], 180000) + assert 'USD' == result['symbol'] + assert result['currencies'] == [{ + 'currency': 'ETH', + 'available': 10.0, + 'balance': 12.0, + 'pending': 2.0, + 'est_btc': 12.0, + }] def test_rpc_start(mocker, default_conf) -> None: From e1921c88490b29735da7d959068bace13ee26c30 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Aug 2018 21:55:48 +0200 Subject: [PATCH 030/175] Fix bug causing /balance to fail --- freqtrade/exchange/__init__.py | 2 +- freqtrade/rpc/rpc.py | 12 +++++++---- freqtrade/tests/rpc/test_rpc.py | 35 ++++++--------------------------- 3 files changed, 15 insertions(+), 34 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index a6ec70636..485599c2c 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -322,7 +322,7 @@ class Exchange(object): return data except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( - f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}') + f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') except ccxt.BaseError as e: raise OperationalException(e) else: diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f58fbae9a..a39469d07 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -13,6 +13,7 @@ import sqlalchemy as sql from numpy import mean, nan_to_num from pandas import DataFrame +from freqtrade import TemporaryError from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.misc import shorten_date from freqtrade.persistence import Trade @@ -271,10 +272,13 @@ class RPC(object): if coin == 'BTC': rate = 1.0 else: - if coin == 'USDT': - rate = 1.0 / self._freqtrade.exchange.get_ticker('BTC/USDT', False)['bid'] - else: - rate = self._freqtrade.exchange.get_ticker(coin + '/BTC', False)['bid'] + try: + if coin == 'USDT': + rate = 1.0 / self._freqtrade.exchange.get_ticker('BTC/USDT', False)['bid'] + else: + rate = self._freqtrade.exchange.get_ticker(coin + '/BTC', False)['bid'] + except TemporaryError: + continue est_btc: float = rate * balance['total'] total = total + est_btc output.append({ diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 01130fccf..c17ab6b2f 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -286,11 +286,12 @@ def test_rpc_balance_handle(default_conf, mocker): 'used': 2.0, }, 'ETH': { - 'free': 0.0, - 'total': 0.0, - 'used': 0.0, + 'free': 1.0, + 'total': 5.0, + 'used': 4.0, } } + # ETH will be skipped due to mocked Error below mocker.patch.multiple( 'freqtrade.fiat_convert.Market', @@ -302,7 +303,8 @@ def test_rpc_balance_handle(default_conf, mocker): mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), - get_balances=MagicMock(return_value=mock_balance) + get_balances=MagicMock(return_value=mock_balance), + get_ticker=MagicMock(side_effect=TemporaryError('Could not load ticker due to xxx')) ) freqtradebot = FreqtradeBot(default_conf) @@ -323,31 +325,6 @@ def test_rpc_balance_handle(default_conf, mocker): }] assert result['total'] == 12.0 - mock_balance = { - 'ETH': { - 'free': 10.0, - 'total': 12.0, - 'used': 2.0, - } - } - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), - get_balances=MagicMock(return_value=mock_balance), - get_ticker=MagicMock(side_effect=TemporaryError('Could not load ticker due to xxx')) - ) - result = rpc._rpc_balance(default_conf['fiat_display_currency']) - assert prec_satoshi(result['total'], 12) - assert prec_satoshi(result['value'], 180000) - assert 'USD' == result['symbol'] - assert result['currencies'] == [{ - 'currency': 'ETH', - 'available': 10.0, - 'balance': 12.0, - 'pending': 2.0, - 'est_btc': 12.0, - }] - def test_rpc_start(mocker, default_conf) -> None: patch_coinmarketcap(mocker) From cef09f49a63676d451571007a202cd52bda42021 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 9 Aug 2018 11:51:38 +0200 Subject: [PATCH 031/175] wait for markets to be loaded before looping in symbols. --- freqtrade/exchange/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 46a56860b..50d759936 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -338,6 +338,7 @@ class Exchange(object): # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ? # loop = asyncio.new_event_loop() # asyncio.set_event_loop(loop) + await self._api_async.load_markets() input_coroutines = [self.async_get_candle_history( symbol, tick_interval) for symbol in pairs] tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) From cb2608522914df2af27f30c51e3a0b6a452e6a54 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 9 Aug 2018 12:47:26 +0200 Subject: [PATCH 032/175] =?UTF-8?q?Moving=20should=5Fnot=5Fupdate=20logic?= =?UTF-8?q?=20to=20async=20function=20per=20pair.=20if=20there=20is=20no?= =?UTF-8?q?=20new=20candle,=20async=20function=20will=20just=20return=20th?= =?UTF-8?q?e=20last=20cached=20candle=20locally=20and=20doesn=E2=80=99t=20?= =?UTF-8?q?hit=20the=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/exchange/__init__.py | 28 ++++++++++++++++++++++++++-- freqtrade/freqtradebot.py | 15 +-------------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 50d759936..449a8270d 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -5,6 +5,7 @@ from random import randint from typing import List, Dict, Tuple, Any, Optional from datetime import datetime from math import floor, ceil +import time import asyncio import ccxt @@ -52,6 +53,12 @@ class Exchange(object): _conf: Dict = {} _cached_ticker: Dict[str, Any] = {} + # Holds last candle refreshed time of each pair + _pairs_last_refreshed_time = {} + + # Holds candles + _cached_klines: Dict[str, Any] = {} + # Holds all open sell orders for dry_run _dry_run_open_orders: Dict[str, Any] = {} @@ -349,8 +356,25 @@ class Exchange(object): since_ms: Optional[int] = None) -> Tuple[str, List]: try: # fetch ohlcv asynchronously - logger.debug("fetching %s ...", pair) - data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) + logger.debug("fetching %s ...", pair) + + # Calculating ticker interval in second + interval_in_seconds = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 + + # If (last update time) + (interval in second) + (1 second) is greater than now + # that means we don't have to hit the API as there is no new candle + # so we fetch it from local cache + if self._pairs_last_refreshed_time.get(pair, 0) + interval_in_seconds + 1 > round(time.time()): + data = self._cached_klines[pair] + else: + data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) + + # keeping last candle time as last refreshed time of the pair + self._pairs_last_refreshed_time[pair] = data[-1][0] / 1000 + + # keeping candles in cache + self._cached_klines[pair] = data + logger.debug("done fetching %s ...", pair) return pair, data diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f4e9c1d5b..d552dc65f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -137,22 +137,9 @@ class FreqtradeBot(object): """ # TODO: maybe add since_ms to use async in the download-script? # TODO: Add tests for this and the async stuff above - - ticker_interval = self.strategy.ticker_interval - interval_in_seconds = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 - - should_not_update = ((self._klines_last_fetched_time + - interval_in_seconds + 1) > round(time.time())) - - if should_not_update: - return False - logger.debug("Refreshing klines for %d pairs", len(pair_list)) datatups = asyncio.get_event_loop().run_until_complete( - self.exchange.async_get_candles_history(pair_list, ticker_interval)) - - # fetching the timestamp of last candle - self._klines_last_fetched_time = datatups[0][1][-1][0] / 1000 + self.exchange.async_get_candles_history(pair_list, self.strategy.ticker_interval)) # updating klines self._klines = {pair: data for (pair, data) in datatups} From d1306a21778d9ad15928bbd4449152d3f890b213 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Aug 2018 19:15:30 +0200 Subject: [PATCH 033/175] Fix failing tests when metadata in `analyze_ticker` is actually used --- freqtrade/tests/test_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index ce144e118..dc030d630 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -14,7 +14,7 @@ def load_dataframe_pair(pairs, strategy): assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] - dataframe = strategy.analyze_ticker(dataframe, pairs[0]) + dataframe = strategy.analyze_ticker(dataframe, {'pair': pairs[0]}) return dataframe From 98730939d4a3b0607774b8668d771505c6b1c287 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 13:02:41 +0200 Subject: [PATCH 034/175] Refactor to use a plain dict * check config-setting first - avoids any call to "candle_seen" eventually --- freqtrade/strategy/interface.py | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 5e0837767..0f03cddb3 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -19,22 +19,6 @@ from freqtrade.persistence import Trade logger = logging.getLogger(__name__) -class CandleAnalyzed: - ''' - Maintains dictionary of the last candle date a pair was processed with - This allows analyze_ticker to test if analysed the candle row in dataframe prior. - To not keep testing the same candle data, which is wasteful in CPU and time - ''' - def __init__(self, last_seen={}): - self.last_seen = last_seen - - def get_last_seen(self, pair): - return self.last_seen.get(pair) - - def set_last_seen(self, pair, candle_date): - self.last_seen[pair] = candle_date - - class SignalType(Enum): """ Enum to distinguish between buy and sell signals @@ -86,9 +70,11 @@ class IStrategy(ABC): # associated ticker interval ticker_interval: str + # Dict to determine if analysis is necessary + candle_seen: Dict[str, datetime] = {} + def __init__(self, config: dict) -> None: self.config = config - self.candleSeen = CandleAnalyzed() @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -133,23 +119,23 @@ class IStrategy(ABC): # Test if seen this pair and last candle before. dataframe = parse_ticker_dataframe(ticker_history) - pair = metadata.get('pair') - last_seen = self.candleSeen.get_last_seen(pair) + pair = str(metadata.get('pair')) - if last_seen != dataframe.iloc[-1]['date'] or self.config.get('ta_on_candle') is False: + if (not self.config.get('ta_on_candle') or + self.candle_seen.get(pair, None) != dataframe.iloc[-1]['date']): # Defs that only make change on new candle data. - logging.info("TA Analysis Launched") + logging.debug("TA Analysis Launched") dataframe = self.advise_indicators(dataframe, metadata) dataframe = self.advise_buy(dataframe, metadata) dataframe = self.advise_sell(dataframe, metadata) - self.candleSeen.set_last_seen(pair=pair, candle_date=dataframe.iloc[-1]['date']) + self.candle_seen[pair] = dataframe.iloc[-1]['date'] else: dataframe.loc['buy'] = 0 dataframe.loc['sell'] = 0 # Other Defs in strategy that want to be called every loop here # twitter_sell = self.watch_twitter_feed(dataframe, metadata) - logging.info("Loop Analysis Launched") + logging.debug("Loop Analysis Launched") return dataframe From 029d61b8c5a3fdcfc0d352dc04bbd1d3ad46f79c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 13:12:12 +0200 Subject: [PATCH 035/175] Add ta_on_candle descripton to support strategy --- docs/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4c43975d7..64e75c51e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -22,8 +22,8 @@ The table below will list all configuration parameters. | `stake_amount` | 0.05 | Yes | Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to 'unlimited' to allow the bot to use all avaliable balance. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes | `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. -| `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. -| `ta_on_candle` | false | No | if set to true indicators are processed each new candle. If false each bot loop, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. +| `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. +| `ta_on_candle` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). From c4e43039f20accf285d201bd6e93177d768697d1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 19:24:00 +0200 Subject: [PATCH 036/175] Allow control from strategy --- freqtrade/strategy/interface.py | 6 +++++- freqtrade/strategy/resolver.py | 9 +++++++++ user_data/strategies/test_strategy.py | 3 +++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 0f03cddb3..494f65547 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -70,6 +70,9 @@ class IStrategy(ABC): # associated ticker interval ticker_interval: str + # run "populate_indicators" only for new candle + ta_on_candle: bool = False + # Dict to determine if analysis is necessary candle_seen: Dict[str, datetime] = {} @@ -121,7 +124,8 @@ class IStrategy(ABC): pair = str(metadata.get('pair')) - if (not self.config.get('ta_on_candle') or + # always run if ta_on_candle is set to true + if (not self.ta_on_candle or self.candle_seen.get(pair, None) != dataframe.iloc[-1]['date']): # Defs that only make change on new candle data. logging.debug("TA Analysis Launched") diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 7aeec300e..1a15ba63f 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -65,6 +65,15 @@ class StrategyResolver(object): else: config['ticker_interval'] = self.strategy.ticker_interval + if 'ta_on_candle' in config: + self.strategy.ta_on_candle = config['ta_on_candle'] + logger.info( + "Override ta_on_candle \'ta_on_candle\' with value in config file: %s.", + config['ta_on_candle'] + ) + else: + config['ta_on_candle'] = self.strategy.ta_on_candle + # Sort and apply type conversions self.strategy.minimal_roi = OrderedDict(sorted( {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 80c238d92..7c3892b77 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -45,6 +45,9 @@ class TestStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' + # run "populate_indicators" only for new candle + ta_on_candle = False + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame From e36067afd35c00799965b3f0aeb14a1ebc7576f6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 19:53:47 +0200 Subject: [PATCH 037/175] refactor candle_seen to private --- freqtrade/strategy/interface.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 494f65547..f8965d440 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -74,10 +74,11 @@ class IStrategy(ABC): ta_on_candle: bool = False # Dict to determine if analysis is necessary - candle_seen: Dict[str, datetime] = {} + _candle_seen: Dict[str, datetime] = {} def __init__(self, config: dict) -> None: self.config = config + self._candle_seen = {} @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -126,16 +127,16 @@ class IStrategy(ABC): # always run if ta_on_candle is set to true if (not self.ta_on_candle or - self.candle_seen.get(pair, None) != dataframe.iloc[-1]['date']): + self._candle_seen.get(pair, None) != dataframe.iloc[-1]['date']): # Defs that only make change on new candle data. logging.debug("TA Analysis Launched") dataframe = self.advise_indicators(dataframe, metadata) dataframe = self.advise_buy(dataframe, metadata) dataframe = self.advise_sell(dataframe, metadata) - self.candle_seen[pair] = dataframe.iloc[-1]['date'] + self._candle_seen[pair] = dataframe.iloc[-1]['date'] else: - dataframe.loc['buy'] = 0 - dataframe.loc['sell'] = 0 + dataframe['buy'] = 0 + dataframe['sell'] = 0 # Other Defs in strategy that want to be called every loop here # twitter_sell = self.watch_twitter_feed(dataframe, metadata) From 4ece5d6d7ac9f20c3f868dc5569e12a9e709bdcc Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:02:24 +0200 Subject: [PATCH 038/175] Add tests for ta_on_candle --- freqtrade/tests/strategy/test_interface.py | 57 ++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index ec4ab0fd4..45f650938 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -105,3 +105,60 @@ def test_tickerdata_to_dataframe(default_conf) -> None: tickerlist = {'UNITTEST/BTC': tick} data = strategy.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed + + +def test_analyze_ticker_default(ticker_history, mocker) -> None: + + ind_mock = MagicMock(side_effect=lambda x, meta: x) + buy_mock = MagicMock(side_effect=lambda x, meta: x) + sell_mock = MagicMock(side_effect=lambda x, meta: x) + mocker.patch.multiple( + 'freqtrade.strategy.interface.IStrategy', + advise_indicators=ind_mock, + advise_buy=buy_mock, + advise_sell=sell_mock, + + ) + strategy = DefaultStrategy({}) + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + assert ind_mock.call_count == 1 + assert buy_mock.call_count == 1 + assert buy_mock.call_count == 1 + + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + # No analysis happens as ta_on_candle is true + assert ind_mock.call_count == 2 + assert buy_mock.call_count == 2 + assert buy_mock.call_count == 2 + + +def test_analyze_ticker_only_once(ticker_history, mocker) -> None: + + ind_mock = MagicMock(side_effect=lambda x, meta: x) + buy_mock = MagicMock(side_effect=lambda x, meta: x) + sell_mock = MagicMock(side_effect=lambda x, meta: x) + mocker.patch.multiple( + 'freqtrade.strategy.interface.IStrategy', + advise_indicators=ind_mock, + advise_buy=buy_mock, + advise_sell=sell_mock, + + ) + strategy = DefaultStrategy({}) + strategy.ta_on_candle = True + + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + assert ind_mock.call_count == 1 + assert buy_mock.call_count == 1 + assert buy_mock.call_count == 1 + + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + # No analysis happens as ta_on_candle is true + assert ind_mock.call_count == 1 + assert buy_mock.call_count == 1 + assert buy_mock.call_count == 1 + # only skipped analyze adds buy and sell columns, otherwise it's all mocked + assert 'buy' in ret + assert 'sell' in ret + assert ret['buy'].sum() == 0 + assert ret['sell'].sum() == 0 From df960241bd7e627c1aeec356c62e4b6f7322c882 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:07:01 +0200 Subject: [PATCH 039/175] Add log-message for skipped candle and tests --- freqtrade/tests/strategy/test_interface.py | 23 ++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 45f650938..75deecda2 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -107,8 +107,8 @@ def test_tickerdata_to_dataframe(default_conf) -> None: assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed -def test_analyze_ticker_default(ticker_history, mocker) -> None: - +def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: + caplog.set_level(logging.DEBUG) ind_mock = MagicMock(side_effect=lambda x, meta: x) buy_mock = MagicMock(side_effect=lambda x, meta: x) sell_mock = MagicMock(side_effect=lambda x, meta: x) @@ -125,15 +125,23 @@ def test_analyze_ticker_default(ticker_history, mocker) -> None: assert buy_mock.call_count == 1 assert buy_mock.call_count == 1 + assert log_has('TA Analysis Launched', caplog.record_tuples) + assert not log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) + caplog.clear() + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) # No analysis happens as ta_on_candle is true assert ind_mock.call_count == 2 assert buy_mock.call_count == 2 assert buy_mock.call_count == 2 + assert log_has('TA Analysis Launched', caplog.record_tuples) + assert not log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) -def test_analyze_ticker_only_once(ticker_history, mocker) -> None: - +def test_analyze_ticker_skip_analyze(ticker_history, mocker, caplog) -> None: + caplog.set_level(logging.DEBUG) ind_mock = MagicMock(side_effect=lambda x, meta: x) buy_mock = MagicMock(side_effect=lambda x, meta: x) sell_mock = MagicMock(side_effect=lambda x, meta: x) @@ -151,6 +159,10 @@ def test_analyze_ticker_only_once(ticker_history, mocker) -> None: assert ind_mock.call_count == 1 assert buy_mock.call_count == 1 assert buy_mock.call_count == 1 + assert log_has('TA Analysis Launched', caplog.record_tuples) + assert not log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) + caplog.clear() ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) # No analysis happens as ta_on_candle is true @@ -162,3 +174,6 @@ def test_analyze_ticker_only_once(ticker_history, mocker) -> None: assert 'sell' in ret assert ret['buy'].sum() == 0 assert ret['sell'].sum() == 0 + assert not log_has('TA Analysis Launched', caplog.record_tuples) + assert log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) From 3b2f161573712c4823e591fe41d6e9e0ce758aea Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:12:45 +0200 Subject: [PATCH 040/175] Add test for ta_on_candle override --- freqtrade/strategy/interface.py | 1 + freqtrade/tests/strategy/test_strategy.py | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index f8965d440..3957139d2 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -135,6 +135,7 @@ class IStrategy(ABC): dataframe = self.advise_sell(dataframe, metadata) self._candle_seen[pair] = dataframe.iloc[-1]['date'] else: + logging.debug("Skippinig TA Analysis for already analyzed candle") dataframe['buy'] = 0 dataframe['sell'] = 0 diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 0cbd9f22c..d45715a69 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -130,7 +130,7 @@ def test_strategy_override_minimal_roi(caplog): assert resolver.strategy.minimal_roi[0] == 0.5 assert ('freqtrade.strategy.resolver', logging.INFO, - 'Override strategy \'minimal_roi\' with value in config file.' + "Override strategy 'minimal_roi' with value in config file." ) in caplog.record_tuples @@ -145,7 +145,7 @@ def test_strategy_override_stoploss(caplog): assert resolver.strategy.stoploss == -0.5 assert ('freqtrade.strategy.resolver', logging.INFO, - 'Override strategy \'stoploss\' with value in config file: -0.5.' + "Override strategy 'stoploss' with value in config file: -0.5." ) in caplog.record_tuples @@ -161,10 +161,25 @@ def test_strategy_override_ticker_interval(caplog): assert resolver.strategy.ticker_interval == 60 assert ('freqtrade.strategy.resolver', logging.INFO, - 'Override strategy \'ticker_interval\' with value in config file: 60.' + "Override strategy 'ticker_interval' with value in config file: 60." ) in caplog.record_tuples +def test_strategy_override_ta_on_candle(caplog): + caplog.set_level(logging.INFO) + + config = { + 'strategy': 'DefaultStrategy', + 'ta_on_candle': True + } + resolver = StrategyResolver(config) + + assert resolver.strategy.ta_on_candle == True + assert ('freqtrade.strategy.resolver', + logging.INFO, + "Override ta_on_candle 'ta_on_candle' with value in config file: True." + ) in caplog.record_tuples + def test_deprecate_populate_indicators(result): default_location = path.join(path.dirname(path.realpath(__file__))) resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', From b008649d790e74f81b114ef82c36b05da4045648 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:13:07 +0200 Subject: [PATCH 041/175] Remove unnecessary quote escaping --- freqtrade/strategy/resolver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 1a15ba63f..75fb99d69 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -44,14 +44,14 @@ class StrategyResolver(object): # Check if we need to override configuration if 'minimal_roi' in config: self.strategy.minimal_roi = config['minimal_roi'] - logger.info("Override strategy \'minimal_roi\' with value in config file.") + logger.info("Override strategy 'minimal_roi' with value in config file.") else: config['minimal_roi'] = self.strategy.minimal_roi if 'stoploss' in config: self.strategy.stoploss = config['stoploss'] logger.info( - "Override strategy \'stoploss\' with value in config file: %s.", config['stoploss'] + "Override strategy 'stoploss' with value in config file: %s.", config['stoploss'] ) else: config['stoploss'] = self.strategy.stoploss @@ -59,7 +59,7 @@ class StrategyResolver(object): if 'ticker_interval' in config: self.strategy.ticker_interval = config['ticker_interval'] logger.info( - "Override strategy \'ticker_interval\' with value in config file: %s.", + "Override strategy 'ticker_interval' with value in config file: %s.", config['ticker_interval'] ) else: @@ -68,7 +68,7 @@ class StrategyResolver(object): if 'ta_on_candle' in config: self.strategy.ta_on_candle = config['ta_on_candle'] logger.info( - "Override ta_on_candle \'ta_on_candle\' with value in config file: %s.", + "Override ta_on_candle 'ta_on_candle' with value in config file: %s.", config['ta_on_candle'] ) else: From 56768f1a617c4fca5c648ffcbe95b3e53b7d7f9c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:17:55 +0200 Subject: [PATCH 042/175] Flake8 in tests ... --- freqtrade/tests/strategy/test_interface.py | 4 ++-- freqtrade/tests/strategy/test_strategy.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 75deecda2..e96dfb024 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -120,7 +120,7 @@ def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: ) strategy = DefaultStrategy({}) - ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) assert ind_mock.call_count == 1 assert buy_mock.call_count == 1 assert buy_mock.call_count == 1 @@ -130,7 +130,7 @@ def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: caplog.record_tuples) caplog.clear() - ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) # No analysis happens as ta_on_candle is true assert ind_mock.call_count == 2 assert buy_mock.call_count == 2 diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index d45715a69..14b1ef1bd 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -174,12 +174,13 @@ def test_strategy_override_ta_on_candle(caplog): } resolver = StrategyResolver(config) - assert resolver.strategy.ta_on_candle == True + assert resolver.strategy.ta_on_candle assert ('freqtrade.strategy.resolver', logging.INFO, "Override ta_on_candle 'ta_on_candle' with value in config file: True." ) in caplog.record_tuples + def test_deprecate_populate_indicators(result): default_location = path.join(path.dirname(path.realpath(__file__))) resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', From e654b76bc8ed24fe6bce239de1cfe6dd927de6b7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Aug 2018 09:44:03 +0200 Subject: [PATCH 043/175] Fix async test --- freqtrade/tests/exchange/test_exchange.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 3a7573f70..ab25a5a7a 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -584,11 +584,15 @@ async def test_async_get_candles_history(default_conf, mocker): async def async_fetch_ohlcv(pair, timeframe, since): return tick + async def async_load_markets(): + return {} + exchange = get_patched_exchange(mocker, default_conf) # Monkey-patch async function exchange._api_async.fetch_ohlcv = async_fetch_ohlcv - exchange = Exchange(default_conf) + exchange._api_async.load_markets = async_load_markets + pairs = ['ETH/BTC', 'XRP/BTC'] res = await exchange.async_get_candles_history(pairs, "5m") assert type(res) is list From 36f05af79ad112b6d7dcaf6ecc38cd1c91eddbed Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Aug 2018 09:44:15 +0200 Subject: [PATCH 044/175] sort fetch_olvhc result, refactor some * add exception for since_ms - if this is set it should always download --- freqtrade/exchange/__init__.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 449a8270d..3a6fdcf35 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -54,7 +54,7 @@ class Exchange(object): _cached_ticker: Dict[str, Any] = {} # Holds last candle refreshed time of each pair - _pairs_last_refreshed_time = {} + _pairs_last_refresh_time = {} # Holds candles _cached_klines: Dict[str, Any] = {} @@ -359,18 +359,26 @@ class Exchange(object): logger.debug("fetching %s ...", pair) # Calculating ticker interval in second - interval_in_seconds = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 + interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 - # If (last update time) + (interval in second) + (1 second) is greater than now + # If (last update time) + (interval in second) is greater or equal than now # that means we don't have to hit the API as there is no new candle # so we fetch it from local cache - if self._pairs_last_refreshed_time.get(pair, 0) + interval_in_seconds + 1 > round(time.time()): + if (not since_ms and + self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= + int(time.time())): data = self._cached_klines[pair] - else: - data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) + else: + data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, + since=since_ms) + + # Because some exchange sort Tickers ASC and other DESC. + # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) + # when GDAX returns a list of tickers DESC (newest first, oldest last) + data = sorted(data, key=lambda x: x[0]) # keeping last candle time as last refreshed time of the pair - self._pairs_last_refreshed_time[pair] = data[-1][0] / 1000 + self._pairs_last_refresh_time[pair] = data[-1][0] // 1000 # keeping candles in cache self._cached_klines[pair] = data From 8a0fc888d64510d5946c647e3fd0a5965c0013b8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Aug 2018 09:48:54 +0200 Subject: [PATCH 045/175] log if using cached data --- freqtrade/exchange/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 3a6fdcf35..bd4745a52 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -368,6 +368,7 @@ class Exchange(object): self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= int(time.time())): data = self._cached_klines[pair] + logger.debug("Using cached klines data for %s ...", pair) else: data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) From e34f2abc3afa78998d4f9fa10c595204b0eaba5a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Aug 2018 09:56:54 +0200 Subject: [PATCH 046/175] Add some typehints --- freqtrade/exchange/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index bd4745a52..6165b7493 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -54,7 +54,7 @@ class Exchange(object): _cached_ticker: Dict[str, Any] = {} # Holds last candle refreshed time of each pair - _pairs_last_refresh_time = {} + _pairs_last_refresh_time : Dict[str, int] = {} # Holds candles _cached_klines: Dict[str, Any] = {} @@ -341,7 +341,8 @@ class Exchange(object): logger.info("returning cached ticker-data for %s", pair) return self._cached_ticker[pair] - async def async_get_candles_history(self, pairs, tick_interval) -> List[Tuple[str, List]]: + async def async_get_candles_history(self, pairs: List[str], + tick_interval: str) -> List[Tuple[str, List]]: # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ? # loop = asyncio.new_event_loop() # asyncio.set_event_loop(loop) From 74d6816a1a0809e7b981748201216f8aea1eeb6a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Aug 2018 10:19:26 +0200 Subject: [PATCH 047/175] Fix some comments --- freqtrade/exchange/__init__.py | 2 +- freqtrade/optimize/__init__.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 6165b7493..0cc707772 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -54,7 +54,7 @@ class Exchange(object): _cached_ticker: Dict[str, Any] = {} # Holds last candle refreshed time of each pair - _pairs_last_refresh_time : Dict[str, int] = {} + _pairs_last_refresh_time: Dict[str, int] = {} # Holds candles _cached_klines: Dict[str, Any] = {} diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 8d5350fe5..502407f07 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -191,19 +191,18 @@ def download_backtesting_testdata(datadir: str, timerange: Optional[TimeRange] = None) -> None: """ - Download the latest ticker intervals from the exchange for the pairs passed in parameters + Download the latest ticker intervals from the exchange for the pair passed in parameters The data is downloaded starting from the last correct ticker interval data that - esists in a cache. If timerange starts earlier than the data in the cache, + exists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data - :param pairs: list of pairs to download + :param pair: pair to download :param tick_interval: ticker interval :param timerange: range of time to download :return: None """ - path = make_testdata_path(datadir) filepair = pair.replace("/", "_") filename = os.path.join(path, f'{filepair}-{tick_interval}.json') From a107c4c7b4203c2f0232faf6f6040d03e7477293 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Aug 2018 11:08:28 +0200 Subject: [PATCH 048/175] Download using asyncio --- freqtrade/exchange/__init__.py | 36 +++++++++++++++++++++++++++++-- freqtrade/optimize/__init__.py | 4 ++-- scripts/download_backtest_data.py | 9 ++++---- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 0cc707772..2cfcfbde7 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -142,6 +142,7 @@ class Exchange(object): try: markets = self._api.load_markets() + asyncio.get_event_loop().run_until_complete(self._api_async.load_markets()) except ccxt.BaseError as e: logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e) return @@ -341,12 +342,43 @@ class Exchange(object): logger.info("returning cached ticker-data for %s", pair) return self._cached_ticker[pair] + def get_history(self, pair: str, tick_interval: str, + since_ms: int) -> List: + """ + Gets candle history using asyncio and returns the list of candles. + Handles all async doing. + """ + return asyncio.get_event_loop().run_until_complete( + self._async_get_history(pair=pair, tick_interval=tick_interval, + since_ms=since_ms)) + + async def _async_get_history(self, pair: str, + tick_interval: str, + since_ms: int) -> List: + # Assume exchange returns 500 candles + _LIMIT = 500 + + one_call = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 * _LIMIT * 1000 + logger.debug("one_call: %s", one_call) + input_coroutines = [self.async_get_candle_history( + pair, tick_interval, since) for since in + range(since_ms, int(time.time() * 1000), one_call)] + tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) + + # Combine tickers + data = [] + for tick in tickers: + if tick[0] == pair: + data.extend(tick[1]) + logger.info("downloaded %s with length %s.", pair, len(data)) + return data + async def async_get_candles_history(self, pairs: List[str], tick_interval: str) -> List[Tuple[str, List]]: # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ? # loop = asyncio.new_event_loop() # asyncio.set_event_loop(loop) - await self._api_async.load_markets() + # await self._api_async.load_markets() input_coroutines = [self.async_get_candle_history( symbol, tick_interval) for symbol in pairs] tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) @@ -357,7 +389,7 @@ class Exchange(object): since_ms: Optional[int] = None) -> Tuple[str, List]: try: # fetch ohlcv asynchronously - logger.debug("fetching %s ...", pair) + logger.debug("fetching %s since %s ...", pair, since_ms) # Calculating ticker interval in second interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 502407f07..49b286fe8 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -218,8 +218,8 @@ def download_backtesting_testdata(datadir: str, logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') - new_data = exchange.get_candle_history(pair=pair, tick_interval=tick_interval, - since_ms=since_ms) + new_data = exchange.get_history(pair=pair, tick_interval=tick_interval, + since_ms=since_ms) data.extend(new_data) logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 686098f94..27c4c1e1c 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""This script generate json data from bittrex""" +"""This script generate json data""" import json import sys from pathlib import Path @@ -52,9 +52,10 @@ exchange = Exchange({'key': '', 'stake_currency': '', 'dry_run': True, 'exchange': { - 'name': args.exchange, - 'pair_whitelist': [] - } + 'name': args.exchange, + 'pair_whitelist': [], + 'ccxt_rate_limit': False + } }) pairs_not_available = [] From a852d2ff32627ac1ffc56e3038319d5511f675c0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Aug 2018 11:15:02 +0200 Subject: [PATCH 049/175] default since_ms to 30 days if no timerange is given --- freqtrade/exchange/__init__.py | 2 +- freqtrade/optimize/__init__.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 2cfcfbde7..1df92b874 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -366,7 +366,7 @@ class Exchange(object): tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) # Combine tickers - data = [] + data: List = [] for tick in tickers: if tick[0] == pair: data.extend(tick[1]) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 49b286fe8..4332f84a4 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -218,8 +218,11 @@ def download_backtesting_testdata(datadir: str, logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') + # Default since_ms to 30 days if nothing is given new_data = exchange.get_history(pair=pair, tick_interval=tick_interval, - since_ms=since_ms) + since_ms=since_ms if since_ms + else + int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000) data.extend(new_data) logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) From fce071843dcb370e8e46a7783249d07d13ebaa17 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Aug 2018 13:04:43 +0200 Subject: [PATCH 050/175] Move async-load to seperate function --- freqtrade/exchange/__init__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 1df92b874..e8889ca17 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -78,6 +78,7 @@ class Exchange(object): self._api = self._init_ccxt(exchange_config) self._api_async = self._init_ccxt(exchange_config, ccxt_async) + self._load_async_markets() logger.info('Using Exchange "%s"', self.name) # Check if all pairs are available @@ -132,6 +133,15 @@ class Exchange(object): "Please check your config.json") raise OperationalException(f'Exchange {name} does not provide a sandbox api') + def _load_async_markets(self) -> None: + try: + if self._api_async: + asyncio.get_event_loop().run_until_complete(self._api_async.load_markets()) + + except ccxt.BaseError as e: + logger.warning('Could not load async markets. Reason: %s', e) + return + def validate_pairs(self, pairs: List[str]) -> None: """ Checks if all given pairs are tradable on the current exchange. @@ -142,7 +152,6 @@ class Exchange(object): try: markets = self._api.load_markets() - asyncio.get_event_loop().run_until_complete(self._api_async.load_markets()) except ccxt.BaseError as e: logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e) return From 88e85e8d33083b4e7efa9147a9b415d1416d6f93 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Aug 2018 13:11:04 +0200 Subject: [PATCH 051/175] fix tests - move load_async_markets call to validate_pairs --- freqtrade/exchange/__init__.py | 2 +- freqtrade/tests/exchange/test_exchange.py | 16 +++++++++++++--- freqtrade/tests/optimize/test_optimize.py | 16 ++++++++-------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index e8889ca17..de1310751 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -78,7 +78,6 @@ class Exchange(object): self._api = self._init_ccxt(exchange_config) self._api_async = self._init_ccxt(exchange_config, ccxt_async) - self._load_async_markets() logger.info('Using Exchange "%s"', self.name) # Check if all pairs are available @@ -152,6 +151,7 @@ class Exchange(object): try: markets = self._api.load_markets() + self._load_async_markets() except ccxt.BaseError as e: logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e) return diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index ab25a5a7a..8fa7a6fec 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -13,6 +13,10 @@ from freqtrade.exchange import API_RETRY_COUNT, Exchange from freqtrade.tests.conftest import get_patched_exchange, log_has +async def async_load_markets(): + return {} + + def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): with pytest.raises(TemporaryError): api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) @@ -78,6 +82,7 @@ def test_symbol_amount_prec(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) exchange = Exchange(default_conf) amount = 2.34559 @@ -101,6 +106,7 @@ def test_symbol_price_prec(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) exchange = Exchange(default_conf) price = 2.34559 @@ -122,6 +128,7 @@ def test_set_sandbox(default_conf, mocker): type(api_mock).urls = url_mock mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) exchange = Exchange(default_conf) liveurl = exchange._api.urls['api'] @@ -143,6 +150,7 @@ def test_set_sandbox_exception(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'does not provide a sandbox api'): exchange = Exchange(default_conf) @@ -160,6 +168,7 @@ def test_validate_pairs(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) Exchange(default_conf) @@ -168,6 +177,7 @@ def test_validate_pairs_not_available(default_conf, mocker): api_mock.load_markets = MagicMock(return_value={}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'not available'): Exchange(default_conf) @@ -181,6 +191,7 @@ def test_validate_pairs_not_compatible(default_conf, mocker): default_conf['stake_currency'] = 'ETH' mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'not compatible'): Exchange(default_conf) @@ -193,6 +204,7 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): api_mock.load_markets = MagicMock(return_value={}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available at Binance'): Exchange(default_conf) @@ -212,6 +224,7 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog): api_mock.name = MagicMock(return_value='binance') mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises( OperationalException, @@ -584,9 +597,6 @@ async def test_async_get_candles_history(default_conf, mocker): async def async_fetch_ohlcv(pair, timeframe, since): return tick - async def async_load_markets(): - return {} - exchange = get_patched_exchange(mocker, default_conf) # Monkey-patch async function exchange._api_async.fetch_ohlcv = async_fetch_ohlcv diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 13f65fbf5..77fa3e3b1 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -53,7 +53,7 @@ def _clean_test_file(file: str) -> None: def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json') _backup_file(file, copy_file=True) optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m') @@ -63,7 +63,7 @@ def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json') _backup_file(file, copy_file=True) @@ -74,7 +74,7 @@ def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') _backup_file(file, copy_file=True) optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) @@ -87,7 +87,7 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog, default_co """ Test load_data() with 1 min ticker """ - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') @@ -118,7 +118,7 @@ def test_testdata_path() -> None: def test_download_pairs(ticker_history, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') @@ -261,7 +261,7 @@ def test_load_cached_data_for_updating(mocker) -> None: def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata', side_effect=BaseException('File Error')) exchange = get_patched_exchange(mocker, default_conf) @@ -279,7 +279,7 @@ def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) # Download a 1 min ticker file @@ -304,7 +304,7 @@ def test_download_backtesting_testdata2(mocker, default_conf) -> None: [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] ] json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=tick) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='1m') download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='3m') From e3e79a55fa6fc1baedac19d8ff1b28af78391a97 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Aug 2018 10:16:51 +0200 Subject: [PATCH 052/175] Fix _abc_data pickle error in 3.7 --- freqtrade/strategy/__init__.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index 283426dfa..49b0c45c0 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -12,8 +12,18 @@ def import_strategy(strategy: IStrategy, config: dict) -> IStrategy: Imports given Strategy instance to global scope of freqtrade.strategy and returns an instance of it """ + # Copy all attributes from base class and class - attr = deepcopy({**strategy.__class__.__dict__, **strategy.__dict__}) + + comb = {**strategy.__class__.__dict__, **strategy.__dict__} + + # Delete '_abc_impl' from dict as deepcopy fails on 3.7 with + # `TypeError: can't pickle _abc_data objects`` + # This will only apply to python 3.7 + if '_abc_impl' in comb: + del comb['_abc_impl'] + + attr = deepcopy(comb) # Adjust module name attr['__module__'] = 'freqtrade.strategy' From f7afd9a5ffc354cc0559d9019b66a8bceaee1d6c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Aug 2018 10:37:10 +0200 Subject: [PATCH 053/175] update setup.sh to support 3.7 --- setup.sh | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/setup.sh b/setup.sh index a825ca41f..557bcdd12 100755 --- a/setup.sh +++ b/setup.sh @@ -1,13 +1,31 @@ #!/usr/bin/env bash #encoding=utf8 +# Check which python version is installed +function check_installed_python() { + which python3.7 + if [ $? -eq 0 ]; then + echo "using Python 3.7" + PYTHON=python3.7 + return + fi + + which python3.6 + if [ $? -eq 0 ]; then + echo "using Python 3.6" + PYTHON=python3.6 + return + fi + +} + function updateenv () { echo "-------------------------" echo "Update your virtual env" echo "-------------------------" source .env/bin/activate echo "pip3 install in-progress. Please wait..." - pip3.6 install --quiet --upgrade pip + pip3 install --quiet --upgrade pip pip3 install --quiet -r requirements.txt --upgrade pip3 install --quiet -r requirements.txt pip3 install --quiet -e . @@ -79,7 +97,7 @@ function reset () { fi echo - python3.6 -m venv .env + ${PYTHON} -m venv .env updateenv } @@ -183,7 +201,7 @@ function install () { install_debian else echo "This script does not support your OS." - echo "If you have Python3.6, pip, virtualenv, ta-lib you can continue." + echo "If you have Python3.6 or Python3.7, pip, virtualenv, ta-lib you can continue." echo "Wait 10 seconds to continue the next install steps or use ctrl+c to interrupt this shell." sleep 10 fi @@ -193,7 +211,7 @@ function install () { echo "-------------------------" echo "Run the bot" echo "-------------------------" - echo "You can now use the bot by executing 'source .env/bin/activate; python3.6 freqtrade/main.py'." + echo "You can now use the bot by executing 'source .env/bin/activate; ${PYTHON} freqtrade/main.py'." } function plot () { @@ -214,6 +232,9 @@ function help () { echo " -p,--plot Install dependencies for Plotting scripts." } +# Verify if 3.6 or 3.7 is installed +check_installed_python + case $* in --install|-i) install From 7d72e364aaa066e4e34387db4906611f39ea01c1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Aug 2018 13:08:10 +0200 Subject: [PATCH 054/175] Remove broken ujson loading - replace with variable-based fix --- freqtrade/optimize/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 5c1bd06ab..a2259e19c 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -3,10 +3,11 @@ import gzip try: import ujson as json + _UJSON = True except ImportError: # see mypy/issues/1153 import json # type: ignore -import inspect + _UJSON = False import logging import os from typing import Optional, List, Dict, Tuple, Any @@ -21,7 +22,7 @@ logger = logging.getLogger(__name__) def json_load(data): """Try to load data with ujson""" - if inspect.getfullargspec(json.load)[5].get('precise_float'): + if _UJSON: return json.load(data, precise_float=True) else: return json.load(data) From a0bc17d1ef25effd20c70130ecc4407d03d4fc2d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Aug 2018 13:59:50 +0200 Subject: [PATCH 055/175] Update dockerfile to 3.7.0 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e959b9296..5d1b44f8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6.6-slim-stretch +FROM python:3.7.0-slim-stretch # Install TA-lib RUN apt-get update && apt-get -y install curl build-essential && apt-get clean From 78610bb47f84007718b8dd6dfe1a7672bb1adb8c Mon Sep 17 00:00:00 2001 From: Nullart2 Date: Tue, 14 Aug 2018 18:12:44 +0800 Subject: [PATCH 056/175] mock order_book and additional test --- freqtrade/freqtradebot.py | 1 + freqtrade/tests/conftest.py | 33 +++++++++++++++++++ freqtrade/tests/test_freqtradebot.py | 47 ++++++++++++++++++++++------ 3 files changed, 72 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 55b9c577f..715cbce80 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -253,6 +253,7 @@ class FreqtradeBot(object): logger.info('Getting price from order book') order_book_top = experimental_bid_strategy.get('order_book_top', 1) order_book = self.exchange.get_order_book(pair, order_book_top) + logger.debug('order_book %s', order_book) # top 1 = index 0 order_book_rate = order_book['bids'][order_book_top - 1][0] # if ticker has lower rate, then use ticker ( usefull if down trending ) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index d7f7e96d9..8a40397f3 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -419,6 +419,39 @@ def limit_sell_order(): } +@pytest.fixture +def order_book_l2(): + return MagicMock(return_value={ + 'bids': [ + [0.043936, 10.442], + [0.043935, 31.865], + [0.043933, 11.212], + [0.043928, 0.088], + [0.043925, 10.0], + [0.043921, 10.0], + [0.04392, 37.64], + [0.043899, 0.066], + [0.043885, 0.676], + [0.04387, 22.758] + ], + 'asks': [ + [0.043949, 0.346], + [0.04395, 0.608], + [0.043951, 3.948], + [0.043954, 0.288], + [0.043958, 9.277], + [0.043995, 1.566], + [0.044, 0.588], + [0.044002, 0.992], + [0.044003, 0.095], + [0.04402, 37.64] + ], + 'timestamp': None, + 'datetime': None, + 'nonce': 288004540 + }) + + @pytest.fixture def ticker_history(): return [ diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 4357b573a..85f393b9d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1887,10 +1887,12 @@ def test_get_real_amount_open_trade(default_conf, mocker): assert freqtrade.get_real_amount(trade, order) == amount -def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, markets, mocker): +def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, markets, mocker, + order_book_l2): default_conf['experimental']['check_depth_of_market']['enabled'] = True default_conf['experimental']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 patch_RPCManager(mocker) + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1920,12 +1922,13 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, assert whitelist == default_conf['exchange']['pair_whitelist'] -def test_order_book_depth_of_market_high_delta(default_conf, ticker, - limit_buy_order, fee, markets, mocker): +def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_order, + fee, markets, mocker, order_book_l2): default_conf['experimental']['check_depth_of_market']['enabled'] = True # delta is 100 which is impossible to reach. hence check_depth_of_market will return false default_conf['experimental']['check_depth_of_market']['bids_to_ask_delta'] = 100 patch_RPCManager(mocker) + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1943,11 +1946,12 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, assert trade is None -def test_order_book_bid_strategy1(default_conf) -> None: +def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: """ test if function get_target_bid will return the order book price instead of the ask rate """ + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) default_conf['exchange']['name'] = 'binance' default_conf['experimental']['bid_strategy']['use_order_book'] = True default_conf['experimental']['bid_strategy']['order_book_top'] = 2 @@ -1955,14 +1959,31 @@ def test_order_book_bid_strategy1(default_conf) -> None: default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) - assert freqtrade.get_target_bid('BTC/USDT', {'ask': 200000, 'last': 200000}) != 200000 + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.045, 'last': 0.046}) == 0.043935 -def test_order_book_bid_strategy2(default_conf) -> None: +def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2) -> None: + """ + test if function get_target_bid will return the ask rate (since its value is lower) + instead of the order book rate (even if enabled) + """ + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + default_conf['exchange']['name'] = 'binance' + default_conf['experimental']['bid_strategy']['use_order_book'] = True + default_conf['experimental']['bid_strategy']['order_book_top'] = 2 + default_conf['bid_strategy']['ask_last_balance'] = 0 + default_conf['telegram']['enabled'] = False + + freqtrade = FreqtradeBot(default_conf) + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.042, 'last': 0.046}) == 0.042 + + +def test_order_book_bid_strategy3(default_conf, mocker, order_book_l2) -> None: """ test if function get_target_bid will return ask rate instead of the order book rate """ + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) default_conf['exchange']['name'] = 'binance' default_conf['experimental']['bid_strategy']['use_order_book'] = True default_conf['experimental']['bid_strategy']['order_book_top'] = 1 @@ -1971,10 +1992,14 @@ def test_order_book_bid_strategy2(default_conf) -> None: freqtrade = FreqtradeBot(default_conf) - assert freqtrade.get_target_bid('BTC/USDT', {'ask': 2, 'last': 2}) == 2 + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.03, 'last': 0.029}) == 0.03 -def test_check_depth_of_market_buy(default_conf) -> None: +def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: + """ + test check depth of market + """ + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) default_conf['telegram']['enabled'] = False default_conf['exchange']['name'] = 'binance' default_conf['experimental']['check_depth_of_market']['enabled'] = True @@ -1987,7 +2012,11 @@ def test_check_depth_of_market_buy(default_conf) -> None: def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order, - fee, markets, mocker) -> None: + fee, markets, mocker, order_book_l2) -> None: + """ + test order book ask strategy + """ + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) default_conf['exchange']['name'] = 'binance' default_conf['experimental']['ask_strategy']['use_order_book'] = True default_conf['experimental']['ask_strategy']['order_book_min'] = 1 From 69cc6aa9587f9e1b4d8a0b630911df578aa91b5c Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Aug 2018 15:59:45 +0200 Subject: [PATCH 057/175] Add test to async --- freqtrade/freqtradebot.py | 1 - freqtrade/tests/exchange/test_exchange.py | 5 ---- freqtrade/tests/test_freqtradebot.py | 30 +++++++++++++++++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d552dc65f..52ada40fa 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -135,7 +135,6 @@ class FreqtradeBot(object): """ Refresh tickers asyncronously and return the result. """ - # TODO: maybe add since_ms to use async in the download-script? # TODO: Add tests for this and the async stuff above logger.debug("Refreshing klines for %d pairs", len(pair_list)) datatups = asyncio.get_event_loop().run_until_complete( diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 8fa7a6fec..f6b6b105f 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -624,11 +624,6 @@ async def test_async_get_candles_history(default_conf, mocker): # await exchange.async_get_candles_history('ETH/BTC', "5m") -def test_refresh_tickers(): - # TODO: Implement test for this - pass - - def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): if since: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 3bf6ad037..489392438 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -137,6 +137,36 @@ def test_throttle_with_assets(mocker, default_conf) -> None: assert result == -1 +def test_refresh_tickers(mocker, default_conf, caplog) -> None: + tick = [ + [ + 1511686200000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + async def async_get_candles_history(pairlist, timeframe): + return [(pair, tick) for pair in pairlist] + + caplog.set_level(logging.DEBUG) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade.exchange.async_get_candles_history = async_get_candles_history + + pairs = ['IOTA/ETH', 'XRP/ETH'] + # empty dicts + assert not freqtrade._klines + freqtrade.refresh_tickers(['IOTA/ETH', 'XRP/ETH']) + + assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples) + assert freqtrade._klines + for pair in pairs: + assert freqtrade._klines[pair] + + def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) From 8528143ffa37941eddb8104148991fdf0bae99e8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Aug 2018 19:51:49 +0200 Subject: [PATCH 058/175] Properly close async exchange as requested by ccxt --- freqtrade/exchange/__init__.py | 9 +++++++++ freqtrade/tests/exchange/test_exchange.py | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index de1310751..07ab4d3c6 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -1,6 +1,7 @@ # pragma pylint: disable=W0603 """ Cryptocurrency Exchanges support """ import logging +import inspect from random import randint from typing import List, Dict, Tuple, Any, Optional from datetime import datetime @@ -87,6 +88,14 @@ class Exchange(object): # Check if timeframe is available self.validate_timeframes(config['ticker_interval']) + def __del__(self): + """ + Destructor - clean up async stuff + """ + logger.debug("Exchange object destroyed, closing async loop") + if self._api_async and inspect.iscoroutinefunction(self._api_async.close): + asyncio.get_event_loop().run_until_complete(self._api_async.close()) + def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index f6b6b105f..f379ee689 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -51,6 +51,12 @@ def test_init(default_conf, mocker, caplog): assert log_has('Instance is running with dry_run enabled', caplog.record_tuples) +def test_destroy(default_conf, mocker, caplog): + caplog.set_level(logging.DEBUG) + get_patched_exchange(mocker, default_conf) + assert log_has('Exchange object destroyed, closing async loop', caplog.record_tuples) + + def test_init_exception(default_conf, mocker): default_conf['exchange']['name'] = 'wrong_exchange_name' From 37e504610a44c839889b752fe474719752be5abc Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Aug 2018 20:33:03 +0200 Subject: [PATCH 059/175] refactor private method - improve some async tests --- freqtrade/exchange/__init__.py | 8 +-- freqtrade/tests/exchange/test_exchange.py | 61 +++++++++++++---------- 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 07ab4d3c6..ae96a1d00 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -378,7 +378,7 @@ class Exchange(object): one_call = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 * _LIMIT * 1000 logger.debug("one_call: %s", one_call) - input_coroutines = [self.async_get_candle_history( + input_coroutines = [self._async_get_candle_history( pair, tick_interval, since) for since in range(since_ms, int(time.time() * 1000), one_call)] tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) @@ -397,14 +397,14 @@ class Exchange(object): # loop = asyncio.new_event_loop() # asyncio.set_event_loop(loop) # await self._api_async.load_markets() - input_coroutines = [self.async_get_candle_history( + input_coroutines = [self._async_get_candle_history( symbol, tick_interval) for symbol in pairs] tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) # await self._api_async.close() return tickers - async def async_get_candle_history(self, pair: str, tick_interval: str, - since_ms: Optional[int] = None) -> Tuple[str, List]: + async def _async_get_candle_history(self, pair: str, tick_interval: str, + since_ms: Optional[int] = None) -> Tuple[str, List]: try: # fetch ohlcv asynchronously logger.debug("fetching %s since %s ...", pair, since_ms) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index f379ee689..e881e5590 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -3,7 +3,8 @@ import logging from datetime import datetime from random import randint -from unittest.mock import MagicMock, PropertyMock +import time +from unittest.mock import Mock, MagicMock, PropertyMock import ccxt import pytest @@ -13,6 +14,14 @@ from freqtrade.exchange import API_RETRY_COUNT, Exchange from freqtrade.tests.conftest import get_patched_exchange, log_has +# Source: https://stackoverflow.com/questions/29881236/how-to-mock-asyncio-coroutines +def get_mock_coro(return_value): + async def mock_coro(*args, **kwargs): + return return_value + + return Mock(wraps=mock_coro) + + async def async_load_markets(): return {} @@ -549,10 +558,10 @@ def test_get_ticker(default_conf, mocker): @pytest.mark.asyncio -async def test_async_get_candle_history(default_conf, mocker): +async def test__async_get_candle_history(default_conf, mocker, caplog): tick = [ [ - 1511686200000, # unix timestamp ms + int(time.time() * 1000), # unix timestamp ms 1, # open 2, # high 3, # low @@ -563,28 +572,37 @@ async def test_async_get_candle_history(default_conf, mocker): async def async_fetch_ohlcv(pair, timeframe, since): return tick - + caplog.set_level(logging.DEBUG) exchange = get_patched_exchange(mocker, default_conf) # Monkey-patch async function - exchange._api_async.fetch_ohlcv = async_fetch_ohlcv + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) exchange = Exchange(default_conf) pair = 'ETH/BTC' - res = await exchange.async_get_candle_history(pair, "5m") + res = await exchange._async_get_candle_history(pair, "5m") assert type(res) is tuple assert len(res) == 2 assert res[0] == pair assert res[1] == tick + assert exchange._api_async.fetch_ohlcv.call_count == 1 + assert not log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples) + # test caching + res = await exchange._async_get_candle_history(pair, "5m") + assert exchange._api_async.fetch_ohlcv.call_count == 1 + assert log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples) + # exchange = Exchange(default_conf) await async_ccxt_exception(mocker, default_conf, MagicMock(), - "async_get_candle_history", "fetch_ohlcv", + "_async_get_candle_history", "fetch_ohlcv", pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + # # reinit exchange + # del exchange - api_mock = MagicMock() - with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'): - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - await exchange.async_get_candle_history(pair, "5m") + # api_mock = MagicMock() + # with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'): + # api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) + # exchange = get_patched_exchange(mocker, default_conf, api_mock) + # await exchange._async_get_candle_history(pair, "5m") @pytest.mark.asyncio @@ -600,14 +618,14 @@ async def test_async_get_candles_history(default_conf, mocker): ] ] - async def async_fetch_ohlcv(pair, timeframe, since): - return tick + async def mock_get_candle_hist(pair, tick_interval, since_ms=None): + return (pair, tick) exchange = get_patched_exchange(mocker, default_conf) # Monkey-patch async function - exchange._api_async.fetch_ohlcv = async_fetch_ohlcv + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) - exchange._api_async.load_markets = async_load_markets + exchange._async_get_candle_history = Mock(wraps=mock_get_candle_hist) pairs = ['ETH/BTC', 'XRP/BTC'] res = await exchange.async_get_candles_history(pairs, "5m") @@ -618,16 +636,7 @@ async def test_async_get_candles_history(default_conf, mocker): assert res[0][1] == tick assert res[1][0] == pairs[1] assert res[1][1] == tick - - # await async_ccxt_exception(mocker, default_conf, MagicMock(), - # "async_get_candles_history", "fetch_ohlcv", - # pairs=pairs, tick_interval=default_conf['ticker_interval']) - - # api_mock = MagicMock() - # with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'): - # api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) - # exchange = get_patched_exchange(mocker, default_conf, api_mock) - # await exchange.async_get_candles_history('ETH/BTC', "5m") + assert exchange._async_get_candle_history.call_count == 2 def make_fetch_ohlcv_mock(data): From 67cbbc86f27dc8f71d37680048f182badfccbb2e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Aug 2018 20:35:12 +0200 Subject: [PATCH 060/175] Add test for exception --- freqtrade/tests/exchange/test_exchange.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index e881e5590..04b505584 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -595,14 +595,12 @@ async def test__async_get_candle_history(default_conf, mocker, caplog): await async_ccxt_exception(mocker, default_conf, MagicMock(), "_async_get_candle_history", "fetch_ohlcv", pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) - # # reinit exchange - # del exchange - # api_mock = MagicMock() - # with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'): - # api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) - # exchange = get_patched_exchange(mocker, default_conf, api_mock) - # await exchange._async_get_candle_history(pair, "5m") + api_mock = MagicMock() + with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'): + api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + await exchange._async_get_candle_history(pair, "5m", int((time.time() - 2000) * 1000)) @pytest.mark.asyncio From e37cb49dc26ec94815078d22d7d932be52303351 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Aug 2018 20:42:13 +0200 Subject: [PATCH 061/175] Ad test for async_load_markets --- freqtrade/tests/exchange/test_exchange.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 04b505584..68a9aba23 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -173,6 +173,20 @@ def test_set_sandbox_exception(default_conf, mocker): exchange.set_sandbox(exchange._api, default_conf['exchange'], 'Logname') +def test__load_async_markets(default_conf, mocker, caplog): + exchange = get_patched_exchange(mocker, default_conf) + exchange._api_async.load_markets = get_mock_coro(None) + exchange._load_async_markets() + assert exchange._api_async.load_markets.call_count == 1 + caplog.set_level(logging.DEBUG) + + exchange._api_async.load_markets = Mock(side_effect=ccxt.BaseError("deadbeef")) + exchange._load_async_markets() + + assert log_has('Could not load async markets. Reason: deadbeef', + caplog.record_tuples) + + def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() api_mock.load_markets = MagicMock(return_value={ From 3aa210cf93f781009a9d62aa50d7956ff5d09979 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Aug 2018 20:53:58 +0200 Subject: [PATCH 062/175] Add test for get_history --- freqtrade/tests/exchange/test_exchange.py | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 68a9aba23..376324757 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -571,6 +571,34 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=True) +def test_get_history(default_conf, mocker, caplog): + exchange = get_patched_exchange(mocker, default_conf) + tick = [ + [ + int(time.time() * 1000), # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + pair = 'ETH/BTC' + + async def mock_cacndle_hist(pair, tick_interval, since_ms): + return pair, tick + + exchange._async_get_candle_history = Mock(wraps=mock_cacndle_hist) + # one_call calculation * 1.8 should do 2 calls + since = 5 * 60 * 500 * 1.8 + print(f"since = {since}") + ret = exchange.get_history(pair, "5m", int((time.time() - since) * 1000)) + + assert exchange._async_get_candle_history.call_count == 2 + # Returns twice the above tick + assert len(ret) == 2 + + @pytest.mark.asyncio async def test__async_get_candle_history(default_conf, mocker, caplog): tick = [ From d007ac4b96b3bf34f7e55a07ee53b0f3680e0b19 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Aug 2018 08:37:20 +0200 Subject: [PATCH 063/175] check version explicitly, use "python" in venv --- freqtrade/strategy/__init__.py | 3 ++- setup.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index 49b0c45c0..38a110bd7 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -1,4 +1,5 @@ import logging +import sys from copy import deepcopy from freqtrade.strategy.interface import IStrategy @@ -20,7 +21,7 @@ def import_strategy(strategy: IStrategy, config: dict) -> IStrategy: # Delete '_abc_impl' from dict as deepcopy fails on 3.7 with # `TypeError: can't pickle _abc_data objects`` # This will only apply to python 3.7 - if '_abc_impl' in comb: + if sys.version_info.major == 3 and sys.version_info.minor == 7 and '_abc_impl' in comb: del comb['_abc_impl'] attr = deepcopy(comb) diff --git a/setup.sh b/setup.sh index 557bcdd12..bd58edbee 100755 --- a/setup.sh +++ b/setup.sh @@ -211,7 +211,7 @@ function install () { echo "-------------------------" echo "Run the bot" echo "-------------------------" - echo "You can now use the bot by executing 'source .env/bin/activate; ${PYTHON} freqtrade/main.py'." + echo "You can now use the bot by executing 'source .env/bin/activate; python freqtrade/main.py'." } function plot () { From ca6594cd24fdbb2dcb2791eafdea99c02e0477fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Aug 2018 12:46:45 +0200 Subject: [PATCH 064/175] remove comment, add docstring --- freqtrade/exchange/__init__.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index ae96a1d00..ce562e056 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -13,7 +13,6 @@ import ccxt import ccxt.async_support as ccxt_async import arrow - from freqtrade import constants, OperationalException, DependencyException, TemporaryError logger = logging.getLogger(__name__) @@ -393,14 +392,10 @@ class Exchange(object): async def async_get_candles_history(self, pairs: List[str], tick_interval: str) -> List[Tuple[str, List]]: - # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ? - # loop = asyncio.new_event_loop() - # asyncio.set_event_loop(loop) - # await self._api_async.load_markets() + """Download ohlcv history for pair-list asyncronously """ input_coroutines = [self._async_get_candle_history( symbol, tick_interval) for symbol in pairs] tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) - # await self._api_async.close() return tickers async def _async_get_candle_history(self, pair: str, tick_interval: str, From 76914c2c07e70e9de6d87d270d7c96990ff97104 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Aug 2018 12:57:27 +0200 Subject: [PATCH 065/175] remove todo comment as this is actually done --- freqtrade/freqtradebot.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 52ada40fa..4e9677b35 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -2,6 +2,7 @@ Freqtrade is the main module of this bot. It contains the class Freqtrade() """ +import asyncio import copy import logging import time @@ -9,13 +10,11 @@ import traceback from datetime import datetime from typing import Any, Callable, Dict, List, Optional -import asyncio import arrow import requests from cachetools import TTLCache, cached - from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange @@ -135,12 +134,11 @@ class FreqtradeBot(object): """ Refresh tickers asyncronously and return the result. """ - # TODO: Add tests for this and the async stuff above logger.debug("Refreshing klines for %d pairs", len(pair_list)) datatups = asyncio.get_event_loop().run_until_complete( self.exchange.async_get_candles_history(pair_list, self.strategy.ticker_interval)) - # updating klines + # updating cached klines available to bot self._klines = {pair: data for (pair, data) in datatups} return True From baeffee80d0bfb7df6431d95cf65f54aa2472d9c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Aug 2018 13:18:52 +0200 Subject: [PATCH 066/175] Replace time.time with arrow.utcnow().timestamp arrow is imported already --- freqtrade/exchange/__init__.py | 5 ++--- freqtrade/tests/exchange/test_exchange.py | 11 ++++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index ce562e056..88e403906 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -6,7 +6,6 @@ from random import randint from typing import List, Dict, Tuple, Any, Optional from datetime import datetime from math import floor, ceil -import time import asyncio import ccxt @@ -379,7 +378,7 @@ class Exchange(object): logger.debug("one_call: %s", one_call) input_coroutines = [self._async_get_candle_history( pair, tick_interval, since) for since in - range(since_ms, int(time.time() * 1000), one_call)] + range(since_ms, arrow.utcnow().timestamp * 1000, one_call)] tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) # Combine tickers @@ -412,7 +411,7 @@ class Exchange(object): # so we fetch it from local cache if (not since_ms and self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= - int(time.time())): + arrow.utcnow().timestamp): data = self._cached_klines[pair] logger.debug("Using cached klines data for %s ...", pair) else: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 376324757..ca06a4a70 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -3,9 +3,9 @@ import logging from datetime import datetime from random import randint -import time from unittest.mock import Mock, MagicMock, PropertyMock +import arrow import ccxt import pytest @@ -575,7 +575,7 @@ def test_get_history(default_conf, mocker, caplog): exchange = get_patched_exchange(mocker, default_conf) tick = [ [ - int(time.time() * 1000), # unix timestamp ms + arrow.utcnow().timestamp * 1000, # unix timestamp ms 1, # open 2, # high 3, # low @@ -592,7 +592,7 @@ def test_get_history(default_conf, mocker, caplog): # one_call calculation * 1.8 should do 2 calls since = 5 * 60 * 500 * 1.8 print(f"since = {since}") - ret = exchange.get_history(pair, "5m", int((time.time() - since) * 1000)) + ret = exchange.get_history(pair, "5m", int((arrow.utcnow().timestamp - since) * 1000)) assert exchange._async_get_candle_history.call_count == 2 # Returns twice the above tick @@ -603,7 +603,7 @@ def test_get_history(default_conf, mocker, caplog): async def test__async_get_candle_history(default_conf, mocker, caplog): tick = [ [ - int(time.time() * 1000), # unix timestamp ms + arrow.utcnow().timestamp * 1000, # unix timestamp ms 1, # open 2, # high 3, # low @@ -642,7 +642,8 @@ async def test__async_get_candle_history(default_conf, mocker, caplog): with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - await exchange._async_get_candle_history(pair, "5m", int((time.time() - 2000) * 1000)) + await exchange._async_get_candle_history(pair, "5m", + (arrow.utcnow().timestamp - 2000) * 1000) @pytest.mark.asyncio From 4a8c1209262d48387ec255b27ac1e8de1ea76aa3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Aug 2018 11:35:01 +0200 Subject: [PATCH 067/175] Output min-roi setting when overwriting from config --- freqtrade/strategy/resolver.py | 7 ++++--- freqtrade/tests/strategy/test_strategy.py | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 7aeec300e..5a44a2c57 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -44,14 +44,15 @@ class StrategyResolver(object): # Check if we need to override configuration if 'minimal_roi' in config: self.strategy.minimal_roi = config['minimal_roi'] - logger.info("Override strategy \'minimal_roi\' with value in config file.") + logger.info("Override strategy 'minimal_roi' with value in config file: %s.", + config['minimal_roi']) else: config['minimal_roi'] = self.strategy.minimal_roi if 'stoploss' in config: self.strategy.stoploss = config['stoploss'] logger.info( - "Override strategy \'stoploss\' with value in config file: %s.", config['stoploss'] + "Override strategy 'stoploss' with value in config file: %s.", config['stoploss'] ) else: config['stoploss'] = self.strategy.stoploss @@ -59,7 +60,7 @@ class StrategyResolver(object): if 'ticker_interval' in config: self.strategy.ticker_interval = config['ticker_interval'] logger.info( - "Override strategy \'ticker_interval\' with value in config file: %s.", + "Override strategy 'ticker_interval' with value in config file: %s.", config['ticker_interval'] ) else: diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 0cbd9f22c..ca41d1d39 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -130,7 +130,7 @@ def test_strategy_override_minimal_roi(caplog): assert resolver.strategy.minimal_roi[0] == 0.5 assert ('freqtrade.strategy.resolver', logging.INFO, - 'Override strategy \'minimal_roi\' with value in config file.' + "Override strategy 'minimal_roi' with value in config file: {'0': 0.5}." ) in caplog.record_tuples @@ -145,7 +145,7 @@ def test_strategy_override_stoploss(caplog): assert resolver.strategy.stoploss == -0.5 assert ('freqtrade.strategy.resolver', logging.INFO, - 'Override strategy \'stoploss\' with value in config file: -0.5.' + "Override strategy 'stoploss' with value in config file: -0.5." ) in caplog.record_tuples @@ -161,7 +161,7 @@ def test_strategy_override_ticker_interval(caplog): assert resolver.strategy.ticker_interval == 60 assert ('freqtrade.strategy.resolver', logging.INFO, - 'Override strategy \'ticker_interval\' with value in config file: 60.' + "Override strategy 'ticker_interval' with value in config file: 60." ) in caplog.record_tuples From e6e2799f03803cc3dee46daa5f46d5e98e400a63 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 16 Aug 2018 11:37:31 +0200 Subject: [PATCH 068/175] Keeping cached Klines only in exchange and renaming _cached_klines to klines. --- freqtrade/exchange/__init__.py | 6 +++--- freqtrade/freqtradebot.py | 9 ++++----- freqtrade/tests/test_freqtradebot.py | 6 +++--- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 69ad4130e..f6bc39239 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -56,7 +56,7 @@ class Exchange(object): _pairs_last_refresh_time: Dict[str, int] = {} # Holds candles - _cached_klines: Dict[str, Any] = {} + klines: Dict[str, Any] = {} # Holds all open sell orders for dry_run _dry_run_open_orders: Dict[str, Any] = {} @@ -412,7 +412,7 @@ class Exchange(object): if (not since_ms and self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= arrow.utcnow().timestamp): - data = self._cached_klines[pair] + data = self.klines[pair] logger.debug("Using cached klines data for %s ...", pair) else: data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, @@ -427,7 +427,7 @@ class Exchange(object): self._pairs_last_refresh_time[pair] = data[-1][0] // 1000 # keeping candles in cache - self._cached_klines[pair] = data + self.klines[pair] = data logger.debug("done fetching %s ...", pair) return pair, data diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index adff27b1d..40c665261 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -55,8 +55,6 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self._init_modules() - self._klines: Dict[str, List[Dict]] = {} - self._klines_last_fetched_time = 0 def _init_modules(self) -> None: """ @@ -173,7 +171,8 @@ class FreqtradeBot(object): self.exchange.async_get_candles_history(pair_list, self.strategy.ticker_interval)) # updating cached klines available to bot - self._klines = {pair: data for (pair, data) in datatups} + #self.exchange.klines = {pair: data for (pair, data) in datatups} + # self.exchange.klines = datatups return True @@ -385,7 +384,7 @@ class FreqtradeBot(object): # running get_signal on historical data fetched # to find buy signals for _pair in whitelist: - (buy, sell) = self.strategy.get_signal(_pair, interval, self._klines.get(_pair)) + (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) if buy and not sell: return self.execute_buy(_pair, stake_amount) @@ -551,7 +550,7 @@ class FreqtradeBot(object): (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): - ticker = self._klines.get(trade.pair) + ticker = self.exchange.klines.get(trade.pair) (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, ticker) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 6e77f6341..42b348892 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -158,13 +158,13 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None: pairs = ['IOTA/ETH', 'XRP/ETH'] # empty dicts - assert not freqtrade._klines + assert not freqtrade.exchange.klines freqtrade.refresh_tickers(['IOTA/ETH', 'XRP/ETH']) assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples) - assert freqtrade._klines + assert freqtrade.exchange.klines for pair in pairs: - assert freqtrade._klines[pair] + assert freqtrade.exchange.klines[pair] def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: From ff8ed564f1d4c930aea39cf376e52765d1f6a220 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Aug 2018 12:15:09 +0200 Subject: [PATCH 069/175] Refactor refresh_pairs to exchange and fix tests --- freqtrade/exchange/__init__.py | 10 +++++++ freqtrade/freqtradebot.py | 17 +----------- freqtrade/tests/exchange/test_exchange.py | 27 +++++++++++++++++++ freqtrade/tests/test_freqtradebot.py | 32 +---------------------- 4 files changed, 39 insertions(+), 47 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index f6bc39239..1f9f147d8 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -389,6 +389,16 @@ class Exchange(object): logger.info("downloaded %s with length %s.", pair, len(data)) return data + def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> bool: + """ + Refresh tickers asyncronously and return the result. + """ + logger.debug("Refreshing klines for %d pairs", len(pair_list)) + asyncio.get_event_loop().run_until_complete( + self.async_get_candles_history(pair_list, ticker_interval)) + + return True + async def async_get_candles_history(self, pairs: List[str], tick_interval: str) -> List[Tuple[str, List]]: """Download ohlcv history for pair-list asyncronously """ diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 40c665261..3cb7ade9a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -2,7 +2,6 @@ Freqtrade is the main module of this bot. It contains the class Freqtrade() """ -import asyncio import copy import logging import time @@ -162,20 +161,6 @@ class FreqtradeBot(object): time.sleep(duration) return result - def refresh_tickers(self, pair_list: List[str]) -> bool: - """ - Refresh tickers asyncronously and return the result. - """ - logger.debug("Refreshing klines for %d pairs", len(pair_list)) - datatups = asyncio.get_event_loop().run_until_complete( - self.exchange.async_get_candles_history(pair_list, self.strategy.ticker_interval)) - - # updating cached klines available to bot - #self.exchange.klines = {pair: data for (pair, data) in datatups} - # self.exchange.klines = datatups - - return True - def _process(self, nb_assets: Optional[int] = 0) -> bool: """ Queries the persistence layer for open trades and handles them, @@ -197,7 +182,7 @@ class FreqtradeBot(object): self.config['exchange']['pair_whitelist'] = final_list # Refreshing candles - self.refresh_tickers(final_list) + self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval) # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index ca06a4a70..a9b786cb8 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -599,6 +599,33 @@ def test_get_history(default_conf, mocker, caplog): assert len(ret) == 2 +def test_refresh_tickers(mocker, default_conf, caplog) -> None: + tick = [ + [ + 1511686200000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + caplog.set_level(logging.DEBUG) + exchange = get_patched_exchange(mocker, default_conf) + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + + pairs = ['IOTA/ETH', 'XRP/ETH'] + # empty dicts + assert not exchange.klines + exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m') + + assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples) + assert exchange.klines + for pair in pairs: + assert exchange.klines[pair] + + @pytest.mark.asyncio async def test__async_get_candle_history(default_conf, mocker, caplog): tick = [ diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 42b348892..5764c5b0f 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -44,7 +44,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: """ freqtrade.strategy.get_signal = lambda e, s, t: value freqtrade.exchange.get_candle_history = lambda p, i: None - freqtrade.refresh_tickers = lambda i: True + freqtrade.exchange.refresh_tickers = lambda p, i: True def patch_RPCManager(mocker) -> MagicMock: @@ -137,36 +137,6 @@ def test_throttle_with_assets(mocker, default_conf) -> None: assert result == -1 -def test_refresh_tickers(mocker, default_conf, caplog) -> None: - tick = [ - [ - 1511686200000, # unix timestamp ms - 1, # open - 2, # high - 3, # low - 4, # close - 5, # volume (in quote currency) - ] - ] - - async def async_get_candles_history(pairlist, timeframe): - return [(pair, tick) for pair in pairlist] - - caplog.set_level(logging.DEBUG) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - freqtrade.exchange.async_get_candles_history = async_get_candles_history - - pairs = ['IOTA/ETH', 'XRP/ETH'] - # empty dicts - assert not freqtrade.exchange.klines - freqtrade.refresh_tickers(['IOTA/ETH', 'XRP/ETH']) - - assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples) - assert freqtrade.exchange.klines - for pair in pairs: - assert freqtrade.exchange.klines[pair] - - def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) From 16fa877b67259cfc5096ee6a625f6138135cde0f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Aug 2018 13:15:46 +0200 Subject: [PATCH 070/175] Remove verbosity of trying backup tables - properly log if databasemigration happened --- freqtrade/persistence.py | 4 +++- freqtrade/tests/test_persistence.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 80d49b895..c26d74015 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -79,10 +79,12 @@ def check_migrate(engine) -> None: table_back_name = 'trades_bak' for i, table_back_name in enumerate(tabs): table_back_name = f'trades_bak{i}' - logger.info(f'trying {table_back_name}') + logger.debug(f'trying {table_back_name}') # Check for latest column if not has_column(cols, 'ticker_interval'): + logger.info(f'Running database migration - backup available as {table_back_name}') + fee_open = get_column_def(cols, 'fee_open', 'fee') fee_close = get_column_def(cols, 'fee_close', 'fee') open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index e52500071..7584537e2 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring, C0103 from unittest.mock import MagicMock +import logging import pytest from sqlalchemy import create_engine @@ -403,6 +404,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): """ Test Database migration (starting with new pairformat) """ + caplog.set_level(logging.DEBUG) amount = 103.223 # Always create all columns apart from the last! create_table_old = """CREATE TABLE IF NOT EXISTS "trades" ( @@ -471,12 +473,15 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert trade.ticker_interval is None assert log_has("trying trades_bak1", caplog.record_tuples) assert log_has("trying trades_bak2", caplog.record_tuples) + assert log_has("Running database migration - backup available as trades_bak2", + caplog.record_tuples) def test_migrate_mid_state(mocker, default_conf, fee, caplog): """ Test Database migration (starting with new pairformat) """ + caplog.set_level(logging.DEBUG) amount = 103.223 create_table_old = """CREATE TABLE IF NOT EXISTS "trades" ( id INTEGER NOT NULL, @@ -530,6 +535,8 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 assert log_has("trying trades_bak0", caplog.record_tuples) + assert log_has("Running database migration - backup available as trades_bak0", + caplog.record_tuples) def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): From dc41a19f995d4c3f6af58aa26f198d9090652156 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 16 Aug 2018 14:27:06 +0200 Subject: [PATCH 071/175] Update ccxt from 1.17.126 to 1.17.132 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c2b90fff7..4fad3a77b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.126 +ccxt==1.17.132 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From d1c5eebff2b95f7443b43505f3fd18297fbd4eee Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Aug 2018 06:50:36 +0200 Subject: [PATCH 072/175] Add explicit test on handling min_roi_reached --- freqtrade/tests/strategy/test_interface.py | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index ec4ab0fd4..2c54e492a 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -8,6 +8,7 @@ from pandas import DataFrame from freqtrade.arguments import TimeRange from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.persistence import Trade from freqtrade.tests.conftest import get_patched_exchange, log_has from freqtrade.strategy.default_strategy import DefaultStrategy @@ -105,3 +106,26 @@ def test_tickerdata_to_dataframe(default_conf) -> None: tickerlist = {'UNITTEST/BTC': tick} data = strategy.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed + + +def test_min_roi_reached(default_conf, fee) -> None: + strategy = DefaultStrategy(default_conf) + strategy.minimal_roi = {0: 0.1, 20: 0.05, 55: 0.01} + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + open_date=arrow.utcnow().shift(hours=-1).datetime, + fee_open=fee.return_value, + fee_close=fee.return_value, + exchange='bittrex', + open_rate=1, + ) + + assert not strategy.min_roi_reached(trade, 0.01, arrow.utcnow().shift(minutes=-55).datetime) + assert strategy.min_roi_reached(trade, 0.12, arrow.utcnow().shift(minutes=-55).datetime) + + assert not strategy.min_roi_reached(trade, 0.04, arrow.utcnow().shift(minutes=-39).datetime) + assert strategy.min_roi_reached(trade, 0.06, arrow.utcnow().shift(minutes=-39).datetime) + + assert not strategy.min_roi_reached(trade, -0.01, arrow.utcnow().shift(minutes=-1).datetime) + assert strategy.min_roi_reached(trade, 0.02, arrow.utcnow().shift(minutes=-1).datetime) From 56188f2f679aaaa2d1f699f783c31dadef434dd1 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 17 Aug 2018 14:27:07 +0200 Subject: [PATCH 073/175] Update ccxt from 1.17.132 to 1.17.134 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4fad3a77b..ede96a535 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.132 +ccxt==1.17.134 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From bc22320f776f524c9d7b39189c8e4fef09168eca Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 18 Aug 2018 14:27:07 +0200 Subject: [PATCH 074/175] Update ccxt from 1.17.134 to 1.17.139 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ede96a535..4a5f1ff33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.134 +ccxt==1.17.139 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From d556f669b0e220bb09b68c7ff5b1fee717e14bad Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Aug 2018 21:05:38 +0200 Subject: [PATCH 075/175] Add async retrier --- freqtrade/exchange/__init__.py | 19 +++++++++++++++++++ freqtrade/tests/exchange/test_exchange.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 1f9f147d8..16ef549a0 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -26,6 +26,24 @@ _EXCHANGE_URLS = { } +def retrier_async(f): + async def wrapper(*args, **kwargs): + count = kwargs.pop('count', API_RETRY_COUNT) + try: + return await f(*args, **kwargs) + except (TemporaryError, DependencyException) as ex: + logger.warning('%s() returned exception: "%s"', f.__name__, ex) + if count > 0: + count -= 1 + kwargs.update({'count': count}) + logger.warning('retrying %s() still for %s times', f.__name__, count) + return await wrapper(*args, **kwargs) + else: + logger.warning('Giving up retrying: %s()', f.__name__) + raise ex + return wrapper + + def retrier(f): def wrapper(*args, **kwargs): count = kwargs.pop('count', API_RETRY_COUNT) @@ -407,6 +425,7 @@ class Exchange(object): tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) return tickers + @retrier_async async def _async_get_candle_history(self, pair: str, tick_interval: str, since_ms: Optional[int] = None) -> Tuple[str, List]: try: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index a9b786cb8..61a3f3efc 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -45,7 +45,7 @@ async def async_ccxt_exception(mocker, default_conf, api_mock, fun, mock_ccxt_fu api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) await getattr(exchange, fun)(**kwargs) - assert api_mock.__dict__[mock_ccxt_fun].call_count == 1 + assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1 with pytest.raises(OperationalException): api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError) From d722c12109281660e07615d178b262620b719ef4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Aug 2018 21:08:59 +0200 Subject: [PATCH 076/175] fix bug in async download script --- freqtrade/exchange/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 16ef549a0..1196cb8fc 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -404,6 +404,8 @@ class Exchange(object): for tick in tickers: if tick[0] == pair: data.extend(tick[1]) + # Sort data again after extending the result - above calls return in "async order" order + data = sorted(data, key=lambda x: x[0]) logger.info("downloaded %s with length %s.", pair, len(data)) return data From 088c54b88c6d8a08c7e0cefc2878834281bf1264 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Aug 2018 09:17:17 +0200 Subject: [PATCH 077/175] remove unnecessary function --- freqtrade/tests/exchange/test_exchange.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 61a3f3efc..367b7d778 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -22,10 +22,6 @@ def get_mock_coro(return_value): return Mock(wraps=mock_coro) -async def async_load_markets(): - return {} - - def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): with pytest.raises(TemporaryError): api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) From 97e9a44fd2a4a7557922544b65646f6f7a7c7216 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 19 Aug 2018 14:28:06 +0200 Subject: [PATCH 078/175] Update ccxt from 1.17.139 to 1.17.146 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4a5f1ff33..c00e4fba0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.139 +ccxt==1.17.146 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 5a0876704a5a124441140c9b98026fc852a3afb7 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 19 Aug 2018 14:28:07 +0200 Subject: [PATCH 079/175] Update pytest from 3.7.1 to 3.7.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c00e4fba0..d6a8dfca1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.0 TA-Lib==0.4.17 -pytest==3.7.1 +pytest==3.7.2 pytest-mock==1.10.0 pytest-cov==2.5.1 tabulate==0.8.2 From 9403248e4d60f3dbf455c3060965d77e7ab3c8a4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Aug 2018 19:48:13 +0200 Subject: [PATCH 080/175] have plot-script use async ticker-refresh --- freqtrade/tests/optimize/test_backtesting.py | 2 +- scripts/plot_dataframe.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 32a5229c0..1625cc5a3 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -110,7 +110,7 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals return pairdata -# use for mock freqtrade.exchange.get_candle_history' +# use for mock ccxt.fetch_ohlvc' def _load_pair_as_ticks(pair, tickfreq): ticks = optimize.load_data(None, ticker_interval=tickfreq, pairs=[pair]) ticks = trim_dictlist(ticks, -201) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index f2f2e0c7f..0f0a3d4cb 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -138,7 +138,8 @@ def plot_analyzed_dataframe(args: Namespace) -> None: tickers = {} if args.live: logger.info('Downloading pair.') - tickers[pair] = exchange.get_candle_history(pair, tick_interval) + exchange.refresh_tickers([pair], tick_interval) + tickers[pair] = exchange.klines[pair] else: tickers = optimize.load_data( datadir=_CONF.get("datadir"), From 694b8be32f8de101a103c0ecda903d10604a8cd4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Aug 2018 19:37:48 +0200 Subject: [PATCH 081/175] Move variables from class to instance --- freqtrade/exchange/__init__.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 1f9f147d8..2f1aca962 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -50,13 +50,6 @@ class Exchange(object): _api: ccxt.Exchange = None _api_async: ccxt_async.Exchange = None _conf: Dict = {} - _cached_ticker: Dict[str, Any] = {} - - # Holds last candle refreshed time of each pair - _pairs_last_refresh_time: Dict[str, int] = {} - - # Holds candles - klines: Dict[str, Any] = {} # Holds all open sell orders for dry_run _dry_run_open_orders: Dict[str, Any] = {} @@ -70,6 +63,14 @@ class Exchange(object): """ self._conf.update(config) + self._cached_ticker: Dict[str, Any] = {} + + # Holds last candle refreshed time of each pair + self._pairs_last_refresh_time: Dict[str, int] = {} + + # Holds candles + self.klines: Dict[str, Any] = {} + if config['dry_run']: logger.info('Instance is running with dry_run enabled') From de0f3e43bf68f9b037875032b03febe1e40e190d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Aug 2018 19:44:40 +0200 Subject: [PATCH 082/175] remove unused mocks --- freqtrade/tests/strategy/test_interface.py | 1 - freqtrade/tests/test_freqtradebot.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index ec4ab0fd4..c821891b1 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -88,7 +88,6 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog): def test_get_signal_handles_exceptions(mocker, default_conf): - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=MagicMock()) exchange = get_patched_exchange(mocker, default_conf) mocker.patch.object( _STRATEGY, 'analyze_ticker', diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 5764c5b0f..fb03cd464 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -43,7 +43,6 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: :return: None """ freqtrade.strategy.get_signal = lambda e, s, t: value - freqtrade.exchange.get_candle_history = lambda p, i: None freqtrade.exchange.refresh_tickers = lambda p, i: True @@ -545,7 +544,6 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), - get_candle_history=MagicMock(return_value=20), get_balance=MagicMock(return_value=20), get_fee=fee, ) From 6d1c82a5faf6e575707e861275ae6dde414868f7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Aug 2018 19:39:22 +0200 Subject: [PATCH 083/175] Remove last refreence to `get_candle_history` --- freqtrade/optimize/backtesting.py | 6 ++--- freqtrade/tests/optimize/test_backtesting.py | 25 +++++++++++++------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9e68318f7..d0b70afc7 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -330,15 +330,15 @@ class Backtesting(object): Run a backtesting end-to-end :return: None """ - data = {} + data: Dict[str, Any] = {} pairs = self.config['exchange']['pair_whitelist'] logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') - for pair in pairs: - data[pair] = self.exchange.get_candle_history(pair, self.ticker_interval) + self.exchange.refresh_tickers(pairs, self.ticker_interval) + data = self.exchange.klines else: logger.info('Using local backtesting data (using whitelist in given config) ...') diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 1625cc5a3..a17867b3a 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -455,7 +455,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) - mocker.patch('freqtrade.exchange.Exchange.get_candle_history') + mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', @@ -490,7 +490,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={})) - mocker.patch('freqtrade.exchange.Exchange.get_candle_history') + mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', @@ -733,9 +733,14 @@ def test_backtest_record(default_conf, fee, mocker): def test_backtest_start_live(default_conf, mocker, caplog): default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', - new=lambda s, n, i: _load_pair_as_ticks(n, i)) - patch_exchange(mocker) + + async def load_pairs(pair, timeframe, since): + return _load_pair_as_ticks(pair, timeframe) + + api_mock = MagicMock() + api_mock.fetch_ohlcv = load_pairs + + patch_exchange(mocker, api_mock) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock()) mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock()) mocker.patch('freqtrade.configuration.open', mocker.mock_open( @@ -776,9 +781,13 @@ def test_backtest_start_live(default_conf, mocker, caplog): def test_backtest_start_multi_strat(default_conf, mocker, caplog): default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', - new=lambda s, n, i: _load_pair_as_ticks(n, i)) - patch_exchange(mocker) + + async def load_pairs(pair, timeframe, since): + return _load_pair_as_ticks(pair, timeframe) + api_mock = MagicMock() + api_mock.fetch_ohlcv = load_pairs + + patch_exchange(mocker, api_mock) backtestmock = MagicMock() mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) gen_table_mock = MagicMock() From a077955efacf6362006007301d535212b458db51 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Aug 2018 19:58:07 +0200 Subject: [PATCH 084/175] update json.load to json_load - followup to #1142 --- freqtrade/optimize/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 5591898fa..0478dbda6 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -177,7 +177,7 @@ def load_cached_data_for_updating(filename: str, # read the cached file if os.path.isfile(filename): with open(filename, "rt") as file: - data = json.load(file) + data = json_load(file) # remove the last item, because we are not sure if it is correct # it could be fetched when the candle was incompleted if data: From 43f73c5aecf048888d3b69fe9c6b5d60125f213f Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 Aug 2018 14:28:06 +0200 Subject: [PATCH 085/175] Update ccxt from 1.17.146 to 1.17.152 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d6a8dfca1..7610f43d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.146 +ccxt==1.17.152 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 8f41e0e190e057b961af2e855093d0ca3ad8fb82 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Aug 2018 20:01:57 +0200 Subject: [PATCH 086/175] Use setting in 'exchange' dict --- freqtrade/strategy/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index aa1e903de..1a2b88c9a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -155,7 +155,7 @@ class IStrategy(ABC): # Check if dataframe is out of date signal_date = arrow.get(latest['date']) interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] - offset = self.config.get('outdated_offset', 5) + offset = self.config.get('exchange', {}).get('outdated_offset', 5) if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + offset))): logger.warning( 'Outdated history for pair %s. Last tick is %s minutes old', From e5707b8a2cd5fea5164d33904903aa2ab364ca79 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 21 Aug 2018 14:28:06 +0200 Subject: [PATCH 087/175] Update ccxt from 1.17.152 to 1.17.157 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7610f43d6..9a6784cf0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.152 +ccxt==1.17.157 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 8a844488d465334973936ba9ea947e2398995ec0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 21 Aug 2018 14:28:08 +0200 Subject: [PATCH 088/175] Update sqlalchemy from 1.2.10 to 1.2.11 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9a6784cf0..4181d0d1b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ccxt==1.17.157 -SQLAlchemy==1.2.10 +SQLAlchemy==1.2.11 python-telegram-bot==10.1.0 arrow==0.12.1 cachetools==2.1.0 From 6e90d482eff02d84cb285493f81f387b16ed1924 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 21 Aug 2018 19:08:21 +0200 Subject: [PATCH 089/175] remove amount_to_lots (deprecated / removed) was removed from ccxt in https://github.com/ccxt/ccxt/commit/527f082e59e1cd3698cb7ae95bdcaae4459ea218 --- freqtrade/exchange/__init__.py | 9 --------- freqtrade/tests/exchange/test_exchange.py | 12 ------------ 2 files changed, 21 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index cd75a7229..60756d84c 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -462,12 +462,3 @@ class Exchange(object): f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') except ccxt.BaseError as e: raise OperationalException(e) - - def get_amount_lots(self, pair: str, amount: float) -> float: - """ - get buyable amount rounding, .. - """ - # validate that markets are loaded before trying to get fee - if not self._api.markets: - self._api.load_markets() - return self._api.amount_to_lots(pair, amount) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 6918e9da1..5ee21c9e4 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -823,15 +823,3 @@ def test_get_fee(default_conf, mocker): ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'get_fee', 'calculate_fee') - - -def test_get_amount_lots(default_conf, mocker): - api_mock = MagicMock() - api_mock.amount_to_lots = MagicMock(return_value=1.0) - api_mock.markets = None - marketmock = MagicMock() - api_mock.load_markets = marketmock - exchange = get_patched_exchange(mocker, default_conf, api_mock) - - assert exchange.get_amount_lots('LTC/BTC', 1.54) == 1 - assert marketmock.call_count == 1 From 4508349d07819f469d6d1c134ea77da5af7778f0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 22 Aug 2018 14:28:07 +0200 Subject: [PATCH 090/175] Update ccxt from 1.17.157 to 1.17.163 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4181d0d1b..467ff4408 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.157 +ccxt==1.17.163 SQLAlchemy==1.2.11 python-telegram-bot==10.1.0 arrow==0.12.1 From ebc072396b1e451b26132216749a573e5033e98f Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 22 Aug 2018 14:28:09 +0200 Subject: [PATCH 091/175] Update numpy from 1.15.0 to 1.15.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 467ff4408..85d2f6b89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ pandas==0.23.4 scikit-learn==0.19.2 scipy==1.1.0 jsonschema==2.6.0 -numpy==1.15.0 +numpy==1.15.1 TA-Lib==0.4.17 pytest==3.7.2 pytest-mock==1.10.0 From 8c0e33753e39116d1140d89e9942ba9fb4dd9291 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 23 Aug 2018 14:28:07 +0200 Subject: [PATCH 092/175] Update ccxt from 1.17.163 to 1.17.170 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 85d2f6b89..e10f88b43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.163 +ccxt==1.17.170 SQLAlchemy==1.2.11 python-telegram-bot==10.1.0 arrow==0.12.1 From ab628c1381e65486fffa22a226fd4e08a98755ad Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 24 Aug 2018 14:28:06 +0200 Subject: [PATCH 093/175] Update ccxt from 1.17.170 to 1.17.176 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e10f88b43..bda14b36b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.170 +ccxt==1.17.176 SQLAlchemy==1.2.11 python-telegram-bot==10.1.0 arrow==0.12.1 From a489a044adca228a68e2d856d2a4af30b909be1e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Aug 2018 13:17:07 +0200 Subject: [PATCH 094/175] Mock Exchange results to avoid random test-failures --- freqtrade/tests/exchange/test_exchange.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 60f51553a..4686176c4 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -515,14 +515,17 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=True) -def test_get_order_book(default_conf, mocker): +def test_get_order_book(default_conf, mocker, order_book_l2): default_conf['exchange']['name'] = 'binance' - exchange = Exchange(default_conf) - order_book = exchange.get_order_book(pair='ETH/BTC', limit=50) + api_mock = MagicMock() + + api_mock.fetch_l2_order_book = order_book_l2 + exchange = get_patched_exchange(mocker, default_conf, api_mock) + order_book = exchange.get_order_book(pair='ETH/BTC', limit=10) assert 'bids' in order_book assert 'asks' in order_book - assert len(order_book['bids']) == 50 - assert len(order_book['asks']) == 50 + assert len(order_book['bids']) == 10 + assert len(order_book['asks']) == 10 def test_get_order_book_exception(default_conf, mocker): From 42587741dd0b6b91d4e837ca546ca3fde256d6d0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Aug 2018 13:21:10 +0200 Subject: [PATCH 095/175] mock exchange to avoid random failures --- freqtrade/tests/test_freqtradebot.py | 36 +++++++++++++++++++++------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 85f393b9d..43ab587ec 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1946,12 +1946,17 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_o assert trade is None -def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: +def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, markets) -> None: """ test if function get_target_bid will return the order book price instead of the ask rate """ - mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_markets=markets, + get_order_book=order_book_l2 + ) default_conf['exchange']['name'] = 'binance' default_conf['experimental']['bid_strategy']['use_order_book'] = True default_conf['experimental']['bid_strategy']['order_book_top'] = 2 @@ -1962,12 +1967,17 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.045, 'last': 0.046}) == 0.043935 -def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2) -> None: +def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) -> None: """ test if function get_target_bid will return the ask rate (since its value is lower) instead of the order book rate (even if enabled) """ - mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_markets=markets, + get_order_book=order_book_l2 + ) default_conf['exchange']['name'] = 'binance' default_conf['experimental']['bid_strategy']['use_order_book'] = True default_conf['experimental']['bid_strategy']['order_book_top'] = 2 @@ -1978,12 +1988,17 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2) -> None: assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.042, 'last': 0.046}) == 0.042 -def test_order_book_bid_strategy3(default_conf, mocker, order_book_l2) -> None: +def test_order_book_bid_strategy3(default_conf, mocker, order_book_l2, markets) -> None: """ test if function get_target_bid will return ask rate instead of the order book rate """ - mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_markets=markets, + get_order_book=order_book_l2 + ) default_conf['exchange']['name'] = 'binance' default_conf['experimental']['bid_strategy']['use_order_book'] = True default_conf['experimental']['bid_strategy']['order_book_top'] = 1 @@ -1995,11 +2010,16 @@ def test_order_book_bid_strategy3(default_conf, mocker, order_book_l2) -> None: assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.03, 'last': 0.029}) == 0.03 -def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: +def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2, markets) -> None: """ test check depth of market """ - mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_markets=markets, + get_order_book=order_book_l2 + ) default_conf['telegram']['enabled'] = False default_conf['exchange']['name'] = 'binance' default_conf['experimental']['check_depth_of_market']['enabled'] = True From 2ee1a2d851fab49b82469df38ebbfc9787270208 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 25 Aug 2018 14:28:06 +0200 Subject: [PATCH 096/175] Update ccxt from 1.17.176 to 1.17.184 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bda14b36b..92957cf94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.176 +ccxt==1.17.184 SQLAlchemy==1.2.11 python-telegram-bot==10.1.0 arrow==0.12.1 From c5efcace4747999c72c02f6218e86e263c7bca20 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Aug 2018 12:49:39 +0200 Subject: [PATCH 097/175] change pip3.6 to pip3 --- docs/installation.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 4de05c121..7d0076673 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -335,11 +335,11 @@ cp config.json.example config.json #### 2. Setup your Python virtual environment (virtualenv) ```bash -python3.6 -m venv .env +python3 -m venv .env source .env/bin/activate -pip3.6 install --upgrade pip -pip3.6 install -r requirements.txt -pip3.6 install -e . +pip3 install --upgrade pip +pip3 install -r requirements.txt +pip3 install -e . ``` #### 3. Run the Bot From fe169483eddb81fe585a13e49c764a193995a476 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 26 Aug 2018 14:28:07 +0200 Subject: [PATCH 098/175] Update ccxt from 1.17.184 to 1.17.188 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 92957cf94..cfee7c5f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.184 +ccxt==1.17.188 SQLAlchemy==1.2.11 python-telegram-bot==10.1.0 arrow==0.12.1 From 1a9c085f10793ccb21ca676f80bce9b161bb0181 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Aug 2018 20:09:12 +0200 Subject: [PATCH 099/175] Restructure install documentation --- docs/installation.md | 151 ++++++++++++++++++------------------------- 1 file changed, 64 insertions(+), 87 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 7d0076673..525c6f187 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -8,7 +8,6 @@ To understand how to set up the bot please read the [Bot Configuration](https:// * [Table of Contents](#table-of-contents) * [Easy Installation - Linux Script](#easy-installation---linux-script) -* [Manual installation](#manual-installation) * [Automatic Installation - Docker](#automatic-installation---docker) * [Custom Linux MacOS Installation](#custom-installation) - [Requirements](#requirements) @@ -56,34 +55,6 @@ Reset parameter will hard reset your branch (only if you are on `master` or `dev Config parameter is a `config.json` configurator. This script will ask you questions to setup your bot and create your `config.json`. -## Manual installation - Linux/MacOS - -The following steps are made for Linux/MacOS environment - -### 1. Clone the repo - -```bash -git clone git@github.com:freqtrade/freqtrade.git -git checkout develop -cd freqtrade -``` - -### 2. Create the config file - -Switch `"dry_run": true,` - -```bash -cp config.json.example config.json -vi config.json -``` - -### 3. Build your docker image and run it - -```bash -docker build -t freqtrade . -docker run --rm -v /etc/localtime:/etc/localtime:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade -``` - ------ ## Automatic Installation - Docker @@ -196,7 +167,7 @@ docker run -d \ freqtrade --db-url sqlite:///tradesv3.sqlite ``` -NOTE: db-url defaults to `sqlite:///tradesv3.sqlite` but it defaults to `sqlite://` if `dry_run=True` is being used. +*Note*: db-url defaults to `sqlite:///tradesv3.sqlite` but it defaults to `sqlite://` if `dry_run=True` is being used. To override this behaviour use a custom db-url value: i.e.: `--db-url sqlite:///tradesv3.dryrun.sqlite` ### 6. Monitor your Docker instance @@ -211,14 +182,15 @@ docker stop freqtrade docker start freqtrade ``` -You do not need to rebuild the image for configuration changes, it will suffice to edit `config.json` and restart the container. +For more information on how to operate Docker, please refer to the [official Docker documentation](https://docs.docker.com/). + +*Note*: You do not need to rebuild the image for configuration changes, it will suffice to edit `config.json` and restart the container. ### 7. Backtest with docker The following assumes that the above steps (1-4) have been completed successfully. Also, backtest-data should be available at `~/.freqtrade/user_data/`. - ``` bash docker run -d \ --name freqtrade \ @@ -238,12 +210,13 @@ Head over to the [Backtesting Documentation](https://github.com/freqtrade/freqtr ## Custom Installation We've included/collected install instructions for Ubuntu 16.04, MacOS, and Windows. These are guidelines and your success may vary with other distros. +OS Specific steps are listed first, the [common](#common) section below is necessary for all systems. ### Requirements Click each one for install guide: -* [Python 3.6.x](http://docs.python-guide.org/en/latest/starting/installation/), note the bot was not tested on Python >= 3.7.x +* [Python >= 3.6.x](http://docs.python-guide.org/en/latest/starting/installation/) * [pip](https://pip.pypa.io/en/stable/installing/) * [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) * [virtualenv](https://virtualenv.pypa.io/en/stable/installation/) (Recommended) @@ -251,7 +224,7 @@ Click each one for install guide: ### Linux - Ubuntu 16.04 -#### 1. Install Python 3.6, Git, and wget +#### Install Python 3.6, Git, and wget ```bash sudo add-apt-repository ppa:jonathonf/python-3.6 @@ -259,7 +232,17 @@ sudo apt-get update sudo apt-get install python3.6 python3.6-venv python3.6-dev build-essential autoconf libtool pkg-config make wget git ``` -#### 2. Install TA-Lib +### MacOS + +#### Install Python 3.6, git, wget and ta-lib + +```bash +brew install python3 git wget +``` + +### common + +#### 1. Install TA-Lib Official webpage: https://mrjbq7.github.io/ta-lib/install.html @@ -275,15 +258,60 @@ cd .. rm -rf ./ta-lib* ``` +*Note*: An already downloaded version of ta-lib is included in the repository, as the sourceforge.net source seems to have problems frequently. + +#### 2. Setup your Python virtual environment (virtualenv) + +*Note*: This step is optional but strongly recommended to keep your system organized + +```bash +python3 -m venv .env +source .env/bin/activate +``` + #### 3. Install FreqTrade Clone the git repository: ```bash git clone https://github.com/freqtrade/freqtrade.git + ``` -#### 4. Configure `freqtrade` as a `systemd` service +Optionally checkout the stable/master branch: + +```bash +git checkout master +``` + +#### 4. Initialize the configuration + +```bash +cd freqtrade +cp config.json.example config.json +``` + +> *To edit the config please refer to [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md).* + +#### 5. Install python dependencies + +``` bash +pip3 install --upgrade pip +pip3 install -r requirements.txt +pip3 install -e . +``` + +#### 6. Run the Bot + +If this is the first time you run the bot, ensure you are running it in Dry-run `"dry_run": true,` otherwise it will start to buy and sell coins. + +```bash +python3.6 ./freqtrade/main.py -c config.json +``` + +*Note*: If you run the bot on a server, you should consider using [Docker](#automatic-installation---docker) a terminal multiplexer like `screen` or [`tmux`](https://en.wikipedia.org/wiki/Tmux) to avoid that the bot is stopped on logout. + +#### 7. [Optional] Configure `freqtrade` as a `systemd` service From the freqtrade repo... copy `freqtrade.service` to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup. @@ -299,57 +327,6 @@ For this to be persistent (run when user is logged out) you'll need to enable `l sudo loginctl enable-linger "$USER" ``` -### MacOS - -#### 1. Install Python 3.6, git, wget and ta-lib - -```bash -brew install python3 git wget ta-lib -``` - -#### 2. Install FreqTrade - -Clone the git repository: - -```bash -git clone https://github.com/freqtrade/freqtrade.git -``` - -Optionally checkout the develop branch: - -```bash -git checkout develop -``` - -### Setup Config and virtual env - -#### 1. Initialize the configuration - -```bash -cd freqtrade -cp config.json.example config.json -``` - -> *To edit the config please refer to [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md).* - -#### 2. Setup your Python virtual environment (virtualenv) - -```bash -python3 -m venv .env -source .env/bin/activate -pip3 install --upgrade pip -pip3 install -r requirements.txt -pip3 install -e . -``` - -#### 3. Run the Bot - -If this is the first time you run the bot, ensure you are running it in Dry-run `"dry_run": true,` otherwise it will start to buy and sell coins. - -```bash -python3.6 ./freqtrade/main.py -c config.json -``` - ------ ## Windows From 188cfc435d8fac9ba1fc47e39abcb69402cdffb8 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 27 Aug 2018 14:28:05 +0200 Subject: [PATCH 100/175] Update ccxt from 1.17.188 to 1.17.194 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cfee7c5f5..007a26323 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.188 +ccxt==1.17.194 SQLAlchemy==1.2.11 python-telegram-bot==10.1.0 arrow==0.12.1 From c99ff78f2f131d2fec272d5393a79a413e8c1d50 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 27 Aug 2018 14:28:07 +0200 Subject: [PATCH 101/175] Update pytest from 3.7.2 to 3.7.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 007a26323..7203d362b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.1 TA-Lib==0.4.17 -pytest==3.7.2 +pytest==3.7.3 pytest-mock==1.10.0 pytest-cov==2.5.1 tabulate==0.8.2 From 19628d317a92e4b66380b344ea938e343d9538cb Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 28 Aug 2018 14:28:06 +0200 Subject: [PATCH 102/175] Update ccxt from 1.17.194 to 1.17.199 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7203d362b..b2f1e3a25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.194 +ccxt==1.17.199 SQLAlchemy==1.2.11 python-telegram-bot==10.1.0 arrow==0.12.1 From 9bce6c5f4837c2651ccb06d28fdb78de87b7b11b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 Aug 2018 19:30:26 +0200 Subject: [PATCH 103/175] Add error-section for windows --- docs/installation.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 525c6f187..820383ff6 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -346,7 +346,7 @@ git clone https://github.com/freqtrade/freqtrade.git copy paste `config.json` to ``\path\freqtrade-develop\freqtrade` -#### install ta-lib +#### Install ta-lib Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). @@ -367,5 +367,17 @@ REM >pip install TA_Lib‑0.4.17‑cp36‑cp36m‑win32.whl > Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/freqtrade/freqtrade/issues/222) +#### Error during installation under Windows + +``` bash +error: Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual C++ Build Tools": http://landinghub.visualstudio.com/visual-cpp-build-tools +``` + +Unfortunately, many packages requiring compilation don't provide a pre-build wheel. It is therefore mandatory to have a C/C++ compiler installed and available for your python environment to use. + +The easiest way is to download install Microsoft Visual Studio Community [here](https://visualstudio.microsoft.com/downloads/) and make sure to install "Common Tools for Visual C++" to enable building c code on Windows. Unfortunately, this is a heavy download / dependency (~4Gb) so you might want to consider WSL or docker first. + +--- + Now you have an environment ready, the next step is [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md)... From c9ee528050cd5d964e50021ad9592d0f74cc2b88 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 Aug 2018 22:06:46 +0200 Subject: [PATCH 104/175] Add section about raspberry / conda to install.md --- docs/installation.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/installation.md b/docs/installation.md index 820383ff6..0fecfcf78 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -232,6 +232,23 @@ sudo apt-get update sudo apt-get install python3.6 python3.6-venv python3.6-dev build-essential autoconf libtool pkg-config make wget git ``` +#### Raspberry Pi / Raspbian + +Before installing FreqTrade on a Raspberry Pi running the official Raspbian Image, make sure you have at least Python 3.6 installed. The default image only provides Python 3.5. Probably the easiest way to get a recent version of python is [miniconda](https://repo.continuum.io/miniconda/). + +The following assumes that miniconda3 is installed and available in your environment, and is installed. +It's recommended to use (mini)conda for this as installation/compilation of `scipy` and `pandas` takes a long time. + +``` bash +conda config --add channels rpi +conda install python=3.6 +conda create -n freqtrade python=3.6 +conda install scipy pandas + +pip install -r requirements.txt +pip install -e . +``` + ### MacOS #### Install Python 3.6, git, wget and ta-lib From b6b89a464fd388b3955990a59f54a5100665bb77 Mon Sep 17 00:00:00 2001 From: Nullart2 Date: Wed, 29 Aug 2018 17:38:43 +0800 Subject: [PATCH 105/175] move order_book config out of experimental --- config.json.example | 29 ++++++++---------- config_full.json.example | 29 ++++++++---------- docs/configuration.md | 14 ++++----- freqtrade/constants.py | 44 ++++++++++++---------------- freqtrade/freqtradebot.py | 24 +++++++-------- freqtrade/tests/conftest.py | 29 ++++++++---------- freqtrade/tests/test_freqtradebot.py | 32 ++++++++++---------- 7 files changed, 92 insertions(+), 109 deletions(-) diff --git a/config.json.example b/config.json.example index c3dc6b5b6..7a0bb6b9b 100644 --- a/config.json.example +++ b/config.json.example @@ -11,7 +11,18 @@ "sell": 30 }, "bid_strategy": { - "ask_last_balance": 0.0 + "ask_last_balance": 0.0, + "use_order_book": false, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9 }, "exchange": { "name": "bittrex", @@ -37,21 +48,7 @@ "experimental": { "use_sell_signal": false, "sell_profit_only": false, - "ignore_roi_if_buy_signal": false, - "check_depth_of_market": { - "enabled": false, - "bids_to_ask_delta": 1 - }, - "bid_strategy": { - "use_order_book": false, - "order_book_top": 2, - "percent_from_top": 0 - }, - "ask_strategy":{ - "use_order_book": false, - "order_book_min": 1, - "order_book_max": 9 - } + "ignore_roi_if_buy_signal": false }, "telegram": { "enabled": true, diff --git a/config_full.json.example b/config_full.json.example index 5a364a93b..717f3c7df 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -20,7 +20,18 @@ "sell": 30 }, "bid_strategy": { - "ask_last_balance": 0.0 + "ask_last_balance": 0.0, + "use_order_book": false, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9 }, "exchange": { "name": "bittrex", @@ -46,21 +57,7 @@ "experimental": { "use_sell_signal": false, "sell_profit_only": false, - "ignore_roi_if_buy_signal": false, - "check_depth_of_market": { - "enabled": false, - "bids_to_ask_delta": 1 - }, - "bid_strategy": { - "use_order_book": false, - "order_book_top": 2, - "percent_from_top": 0 - }, - "ask_strategy":{ - "use_order_book": false, - "order_book_min": 1, - "order_book_max": 9 - }s + "ignore_roi_if_buy_signal": false }, "telegram": { "enabled": true, diff --git a/docs/configuration.md b/docs/configuration.md index 5ffe24556..edc376f90 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -31,6 +31,13 @@ The table below will list all configuration parameters. | `unfilledtimeout.buy` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. | `unfilledtimeout.sell` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. +| `bid_strategy.use_order_book` | false | No | Allows buying of pair using the rates in Order Book Bids. +| `bid_strategy.order_book_top` | 0 | No | Bot will use the top N rate in Order Book Bids. Ie. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids. +| `bid_strategy.check_depth_of_market.enabled` | false | No | Does not buy if the % difference of buy orders and sell orders is met in Order Book. +| `experimental.check_depth_of_market.bids_to_ask_delta` | 0 | No | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. +| `ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks. +| `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. +| `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. @@ -39,13 +46,6 @@ The table below will list all configuration parameters. | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. | `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` -| `experimental.check_depth_of_market` | false | No | Does not sell if the % difference of buy orders and sell orders is met in Order Book. -| `experimental.bids_to_ask_delta` | 0 | No | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. -| `experimental.bid_strategy.use_order_book` | false | No | Allows buying of pair using the rates in Order Book Bids. -| `experimental.bid_strategy.order_book_top` | 0 | No | Bot will use the top N rate in Order Book Bids. Ie. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids. -| `experimental.ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks. -| `experimental.ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. -| `experimental.ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `telegram.enabled` | true | Yes | Enable or not the usage of Telegram. | `telegram.token` | token | No | Your Telegram bot token. Only required if `telegram.enabled` is `true`. | `telegram.chat_id` | chat_id | No | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index b7431af3c..59b1c3ccf 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -78,41 +78,35 @@ CONF_SCHEMA = { 'type': 'number', 'minimum': 0, 'maximum': 1, - 'exclusiveMaximum': False + 'exclusiveMaximum': False, + 'use_order_book': {'type': 'boolean'}, + 'order_book_top': {'type': 'number', 'maximum': 20, 'minimum': 1}, + 'check_depth_of_market': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'bids_to_ask_delta': {'type': 'number', 'minimum': 0}, + } + }, }, }, 'required': ['ask_last_balance'] }, + 'ask_strategy': { + 'type': 'object', + 'properties': { + 'use_order_book': {'type': 'boolean'}, + 'order_book_min': {'type': 'number', 'minimum': 1}, + 'order_book_max': {'type': 'number', 'minimum': 1, 'maximum': 50} + } + }, 'exchange': {'$ref': '#/definitions/exchange'}, 'experimental': { 'type': 'object', 'properties': { 'use_sell_signal': {'type': 'boolean'}, 'sell_profit_only': {'type': 'boolean'}, - 'ignore_roi_if_buy_signal_true': {'type': 'boolean'}, - 'check_depth_of_market': { - 'type': 'object', - 'properties': { - 'enabled': {'type': 'boolean'}, - 'bids_to_ask_delta': {'type': 'number', 'minimum': 0}, - } - }, - 'bid_strategy': { - 'type': 'object', - 'properties': { - 'percent_from_top': {'type': 'number', 'minimum': 0}, - 'use_order_book': {'type': 'boolean'}, - 'order_book_top': {'type': 'number', 'maximum': 20, 'minimum': 1} - } - }, - 'ask_strategy': { - 'type': 'object', - 'properties': { - 'use_order_book': {'type': 'boolean'}, - 'order_book_min': {'type': 'number', 'minimum': 1}, - 'order_book_max': {'type': 'number', 'minimum': 1, 'maximum': 50} - } - } + 'ignore_roi_if_buy_signal_true': {'type': 'boolean'} } }, 'telegram': { diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 715cbce80..1f25fb0e4 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -247,11 +247,11 @@ class FreqtradeBot(object): ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask']) used_rate = ticker_rate - experimental_bid_strategy = self.config.get('experimental', {}).get('bid_strategy', {}) - if 'use_order_book' in experimental_bid_strategy and\ - experimental_bid_strategy.get('use_order_book', False): + config_bid_strategy = self.config.get('bid_strategy', {}) + if 'use_order_book' in config_bid_strategy and\ + config_bid_strategy.get('use_order_book', False): logger.info('Getting price from order book') - order_book_top = experimental_bid_strategy.get('order_book_top', 1) + order_book_top = config_bid_strategy.get('order_book_top', 1) order_book = self.exchange.get_order_book(pair, order_book_top) logger.debug('order_book %s', order_book) # top 1 = index 0 @@ -359,11 +359,11 @@ class FreqtradeBot(object): (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) if buy and not sell: - experimental_check_depth_of_market = self.config.get('experimental', {}).\ + bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\ get('check_depth_of_market', {}) - if (experimental_check_depth_of_market.get('enabled', False)) and\ - (experimental_check_depth_of_market.get('bids_to_ask_delta', 0) > 0): - if self._check_depth_of_market_buy(_pair, experimental_check_depth_of_market): + if (bidstrat_check_depth_of_market.get('enabled', False)) and\ + (bidstrat_check_depth_of_market.get('bids_to_ask_delta', 0) > 0): + if self._check_depth_of_market_buy(_pair, bidstrat_check_depth_of_market): return self.execute_buy(_pair, stake_amount) else: return False @@ -551,12 +551,12 @@ class FreqtradeBot(object): (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, ticker) - experimental_ask_strategy = self.config.get('experimental', {}).get('ask_strategy', {}) - if experimental_ask_strategy.get('use_order_book', False): + config_ask_strategy = self.config.get('ask_strategy', {}) + if config_ask_strategy.get('use_order_book', False): logger.info('Using order book for selling...') # logger.debug('Order book %s',orderBook) - order_book_min = experimental_ask_strategy.get('order_book_min', 1) - order_book_max = experimental_ask_strategy.get('order_book_max', 1) + order_book_min = config_ask_strategy.get('order_book_min', 1) + order_book_max = config_ask_strategy.get('order_book_max', 1) order_book = self.exchange.get_order_book(trade.pair, order_book_max) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 8a40397f3..af9062cab 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -102,7 +102,18 @@ def default_conf(): "sell": 30 }, "bid_strategy": { - "ask_last_balance": 0.0 + "ask_last_balance": 0.0, + "use_order_book": False, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": False, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy": { + "use_order_book": False, + "order_book_min": 1, + "order_book_max": 1 }, "exchange": { "name": "bittrex", @@ -116,22 +127,6 @@ def default_conf(): "NEO/BTC" ] }, - "experimental": { - "check_depth_of_market": { - "enabled": False, - "bids_to_ask_delta": 1 - }, - "bid_strategy": { - "percent_from_top": 0, - "use_order_book": False, - "order_book_top": 1 - }, - "ask_strategy": { - "use_order_book": False, - "order_book_min": 1, - "order_book_max": 1 - } - }, "telegram": { "enabled": True, "token": "token", diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 43ab587ec..389215f6f 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1889,8 +1889,8 @@ def test_get_real_amount_open_trade(default_conf, mocker): def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, markets, mocker, order_book_l2): - default_conf['experimental']['check_depth_of_market']['enabled'] = True - default_conf['experimental']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 + default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True + default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 patch_RPCManager(mocker) mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) mocker.patch.multiple( @@ -1924,9 +1924,9 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_order, fee, markets, mocker, order_book_l2): - default_conf['experimental']['check_depth_of_market']['enabled'] = True + default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True # delta is 100 which is impossible to reach. hence check_depth_of_market will return false - default_conf['experimental']['check_depth_of_market']['bids_to_ask_delta'] = 100 + default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100 patch_RPCManager(mocker) mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) mocker.patch.multiple( @@ -1958,8 +1958,8 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, markets) get_order_book=order_book_l2 ) default_conf['exchange']['name'] = 'binance' - default_conf['experimental']['bid_strategy']['use_order_book'] = True - default_conf['experimental']['bid_strategy']['order_book_top'] = 2 + default_conf['bid_strategy']['use_order_book'] = True + default_conf['bid_strategy']['order_book_top'] = 2 default_conf['bid_strategy']['ask_last_balance'] = 0 default_conf['telegram']['enabled'] = False @@ -1979,8 +1979,8 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) get_order_book=order_book_l2 ) default_conf['exchange']['name'] = 'binance' - default_conf['experimental']['bid_strategy']['use_order_book'] = True - default_conf['experimental']['bid_strategy']['order_book_top'] = 2 + default_conf['bid_strategy']['use_order_book'] = True + default_conf['bid_strategy']['order_book_top'] = 2 default_conf['bid_strategy']['ask_last_balance'] = 0 default_conf['telegram']['enabled'] = False @@ -2000,8 +2000,8 @@ def test_order_book_bid_strategy3(default_conf, mocker, order_book_l2, markets) get_order_book=order_book_l2 ) default_conf['exchange']['name'] = 'binance' - default_conf['experimental']['bid_strategy']['use_order_book'] = True - default_conf['experimental']['bid_strategy']['order_book_top'] = 1 + default_conf['bid_strategy']['use_order_book'] = True + default_conf['bid_strategy']['order_book_top'] = 1 default_conf['bid_strategy']['ask_last_balance'] = 0 default_conf['telegram']['enabled'] = False @@ -2022,12 +2022,12 @@ def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2, markets) ) default_conf['telegram']['enabled'] = False default_conf['exchange']['name'] = 'binance' - default_conf['experimental']['check_depth_of_market']['enabled'] = True + default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True # delta is 100 which is impossible to reach. hence function will return false - default_conf['experimental']['check_depth_of_market']['bids_to_ask_delta'] = 100 + default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100 freqtrade = FreqtradeBot(default_conf) - conf = default_conf['experimental']['check_depth_of_market'] + conf = default_conf['bid_strategy']['check_depth_of_market'] assert freqtrade._check_depth_of_market_buy('ETH/BTC', conf) is False @@ -2038,9 +2038,9 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order """ mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) default_conf['exchange']['name'] = 'binance' - default_conf['experimental']['ask_strategy']['use_order_book'] = True - default_conf['experimental']['ask_strategy']['order_book_min'] = 1 - default_conf['experimental']['ask_strategy']['order_book_max'] = 2 + default_conf['ask_strategy']['use_order_book'] = True + default_conf['ask_strategy']['order_book_min'] = 1 + default_conf['ask_strategy']['order_book_max'] = 2 default_conf['telegram']['enabled'] = False patch_RPCManager(mocker) mocker.patch.multiple( From b659ec00ee495e59f1c055d019abd7e7c57c8282 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 29 Aug 2018 14:28:07 +0200 Subject: [PATCH 106/175] Update ccxt from 1.17.199 to 1.17.205 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b2f1e3a25..fe64ea684 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.199 +ccxt==1.17.205 SQLAlchemy==1.2.11 python-telegram-bot==10.1.0 arrow==0.12.1 From f7b67cec5b50a93bf0bad4e30656b408caf8a47b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Aug 2018 19:16:41 +0200 Subject: [PATCH 107/175] Fix missing docstring --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index edc376f90..757310957 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -34,7 +34,7 @@ The table below will list all configuration parameters. | `bid_strategy.use_order_book` | false | No | Allows buying of pair using the rates in Order Book Bids. | `bid_strategy.order_book_top` | 0 | No | Bot will use the top N rate in Order Book Bids. Ie. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids. | `bid_strategy.check_depth_of_market.enabled` | false | No | Does not buy if the % difference of buy orders and sell orders is met in Order Book. -| `experimental.check_depth_of_market.bids_to_ask_delta` | 0 | No | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. +| `bid_strategy.check_depth_of_market.bids_to_ask_delta` | 0 | No | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. | `ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks. | `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. From ffd4469c1d4317510f8f279a8a16812ae218ef2f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Aug 2018 19:56:38 +0200 Subject: [PATCH 108/175] fix typo, refresh_tickers does not need a return value --- freqtrade/exchange/__init__.py | 4 +--- freqtrade/tests/exchange/test_exchange.py | 4 ++-- freqtrade/tests/test_freqtradebot.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index bcb539996..fbc84c9b9 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -410,7 +410,7 @@ class Exchange(object): logger.info("downloaded %s with length %s.", pair, len(data)) return data - def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> bool: + def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> None: """ Refresh tickers asyncronously and return the result. """ @@ -418,8 +418,6 @@ class Exchange(object): asyncio.get_event_loop().run_until_complete( self.async_get_candles_history(pair_list, ticker_interval)) - return True - async def async_get_candles_history(self, pairs: List[str], tick_interval: str) -> List[Tuple[str, List]]: """Download ohlcv history for pair-list asyncronously """ diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index f752da0a3..3c90f425c 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -581,10 +581,10 @@ def test_get_history(default_conf, mocker, caplog): ] pair = 'ETH/BTC' - async def mock_cacndle_hist(pair, tick_interval, since_ms): + async def mock_candle_hist(pair, tick_interval, since_ms): return pair, tick - exchange._async_get_candle_history = Mock(wraps=mock_cacndle_hist) + exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls since = 5 * 60 * 500 * 1.8 print(f"since = {since}") diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 6971ec0dd..5e982f11a 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -43,7 +43,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: :return: None """ freqtrade.strategy.get_signal = lambda e, s, t: value - freqtrade.exchange.refresh_tickers = lambda p, i: True + freqtrade.exchange.refresh_tickers = lambda p, i: None def patch_RPCManager(mocker) -> MagicMock: From a1bd30aa605228510349c2ec97671eb28af282de Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Aug 2018 19:59:25 +0200 Subject: [PATCH 109/175] Fix documentation string --- docs/configuration.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 757310957..3866effd2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -25,9 +25,9 @@ The table below will list all configuration parameters. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. -| `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). -| `trailing_stoploss_positve` | 0 | No | Changes stop-loss once profit has been reached. -| `trailing_stoploss_positve_offset` | 0 | No | Offset on when to apply `trailing_stoploss_positive`. Percentage value which should be positive. +| `trailing_stop` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). +| `trailing_stop_positve` | 0 | No | Changes stop-loss once profit has been reached. +| `trailing_stop_positve_offset` | 0 | No | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. | `unfilledtimeout.buy` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. | `unfilledtimeout.sell` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. From 35c5d4f5804b211b97889301e93cd872c35b1fba Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 30 Aug 2018 14:28:07 +0200 Subject: [PATCH 110/175] Update ccxt from 1.17.205 to 1.17.210 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fe64ea684..fcffbf887 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.205 +ccxt==1.17.210 SQLAlchemy==1.2.11 python-telegram-bot==10.1.0 arrow==0.12.1 From 3ed97fe5e880dd0cb3ffcad238a9e0fa9f340b20 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 30 Aug 2018 14:28:08 +0200 Subject: [PATCH 111/175] Update python-telegram-bot from 10.1.0 to 11.0.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fcffbf887..0e0685487 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ ccxt==1.17.210 SQLAlchemy==1.2.11 -python-telegram-bot==10.1.0 +python-telegram-bot==11.0.0 arrow==0.12.1 cachetools==2.1.0 requests==2.19.1 From 9560cb80566efb3414ed0bbbc7da5a93318f345f Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 30 Aug 2018 14:28:10 +0200 Subject: [PATCH 112/175] Update pytest from 3.7.3 to 3.7.4 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0e0685487..3a605a663 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.1 TA-Lib==0.4.17 -pytest==3.7.3 +pytest==3.7.4 pytest-mock==1.10.0 pytest-cov==2.5.1 tabulate==0.8.2 From fa5c8e4bb11519ec3302beed5324b7c85ba19635 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Sep 2018 14:28:06 +0200 Subject: [PATCH 113/175] Update ccxt from 1.17.210 to 1.17.216 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3a605a663..5cb061855 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.210 +ccxt==1.17.216 SQLAlchemy==1.2.11 python-telegram-bot==11.0.0 arrow==0.12.1 From cb46aeb73cc277a2883c75ba3feadeb7ad2c132b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Sep 2018 19:50:45 +0200 Subject: [PATCH 114/175] rename variable to be more expressive --- freqtrade/strategy/interface.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 3957139d2..a9838a5cb 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -74,11 +74,11 @@ class IStrategy(ABC): ta_on_candle: bool = False # Dict to determine if analysis is necessary - _candle_seen: Dict[str, datetime] = {} + _last_candle_seen_per_pair: Dict[str, datetime] = {} def __init__(self, config: dict) -> None: self.config = config - self._candle_seen = {} + self._last_candle_seen_per_pair = {} @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -119,7 +119,6 @@ class IStrategy(ABC): add several TA indicators and buy signal to it :return DataFrame with ticker data and indicator data """ - # Test if seen this pair and last candle before. dataframe = parse_ticker_dataframe(ticker_history) @@ -127,13 +126,13 @@ class IStrategy(ABC): # always run if ta_on_candle is set to true if (not self.ta_on_candle or - self._candle_seen.get(pair, None) != dataframe.iloc[-1]['date']): + self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']): # Defs that only make change on new candle data. logging.debug("TA Analysis Launched") dataframe = self.advise_indicators(dataframe, metadata) dataframe = self.advise_buy(dataframe, metadata) dataframe = self.advise_sell(dataframe, metadata) - self._candle_seen[pair] = dataframe.iloc[-1]['date'] + self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date'] else: logging.debug("Skippinig TA Analysis for already analyzed candle") dataframe['buy'] = 0 From d35d3bb38cf9de50b0d6d2a234297d6c8b892cee Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Sep 2018 19:52:40 +0200 Subject: [PATCH 115/175] rename ta_on_candle to process_only_new_candles be more expressive --- docs/configuration.md | 2 +- freqtrade/constants.py | 2 +- freqtrade/strategy/interface.py | 6 +++--- freqtrade/strategy/resolver.py | 10 +++++----- freqtrade/tests/strategy/test_interface.py | 6 +++--- freqtrade/tests/strategy/test_strategy.py | 9 +++++---- 6 files changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 64e75c51e..1d5fc0922 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -23,7 +23,7 @@ The table below will list all configuration parameters. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes | `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. -| `ta_on_candle` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. +| `process_only_new_candles` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 189b78617..92b090432 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -53,7 +53,7 @@ CONF_SCHEMA = { }, 'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT}, 'dry_run': {'type': 'boolean'}, - 'ta_on_candle': {'type': 'boolean'}, + 'process_only_new_candles': {'type': 'boolean'}, 'minimal_roi': { 'type': 'object', 'patternProperties': { diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index a9838a5cb..05ede129e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -71,7 +71,7 @@ class IStrategy(ABC): ticker_interval: str # run "populate_indicators" only for new candle - ta_on_candle: bool = False + process_only_new_candles: bool = False # Dict to determine if analysis is necessary _last_candle_seen_per_pair: Dict[str, datetime] = {} @@ -124,8 +124,8 @@ class IStrategy(ABC): pair = str(metadata.get('pair')) - # always run if ta_on_candle is set to true - if (not self.ta_on_candle or + # always run if process_only_new_candles is set to true + if (not self.process_only_new_candles or self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']): # Defs that only make change on new candle data. logging.debug("TA Analysis Launched") diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 75fb99d69..35aee8d20 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -65,14 +65,14 @@ class StrategyResolver(object): else: config['ticker_interval'] = self.strategy.ticker_interval - if 'ta_on_candle' in config: - self.strategy.ta_on_candle = config['ta_on_candle'] + if 'process_only_new_candles' in config: + self.strategy.process_only_new_candles = config['process_only_new_candles'] logger.info( - "Override ta_on_candle 'ta_on_candle' with value in config file: %s.", - config['ta_on_candle'] + "Override process_only_new_candles 'process_only_new_candles' " + "with value in config file: %s.", config['process_only_new_candles'] ) else: - config['ta_on_candle'] = self.strategy.ta_on_candle + config['process_only_new_candles'] = self.strategy.process_only_new_candles # Sort and apply type conversions self.strategy.minimal_roi = OrderedDict(sorted( diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index e96dfb024..5afffd87f 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -131,7 +131,7 @@ def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: caplog.clear() strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) - # No analysis happens as ta_on_candle is true + # No analysis happens as process_only_new_candles is true assert ind_mock.call_count == 2 assert buy_mock.call_count == 2 assert buy_mock.call_count == 2 @@ -153,7 +153,7 @@ def test_analyze_ticker_skip_analyze(ticker_history, mocker, caplog) -> None: ) strategy = DefaultStrategy({}) - strategy.ta_on_candle = True + strategy.process_only_new_candles = True ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) assert ind_mock.call_count == 1 @@ -165,7 +165,7 @@ def test_analyze_ticker_skip_analyze(ticker_history, mocker, caplog) -> None: caplog.clear() ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) - # No analysis happens as ta_on_candle is true + # No analysis happens as process_only_new_candles is true assert ind_mock.call_count == 1 assert buy_mock.call_count == 1 assert buy_mock.call_count == 1 diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 14b1ef1bd..8a6e7e617 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -165,19 +165,20 @@ def test_strategy_override_ticker_interval(caplog): ) in caplog.record_tuples -def test_strategy_override_ta_on_candle(caplog): +def test_strategy_override_process_only_new_candles(caplog): caplog.set_level(logging.INFO) config = { 'strategy': 'DefaultStrategy', - 'ta_on_candle': True + 'process_only_new_candles': True } resolver = StrategyResolver(config) - assert resolver.strategy.ta_on_candle + assert resolver.strategy.process_only_new_candles assert ('freqtrade.strategy.resolver', logging.INFO, - "Override ta_on_candle 'ta_on_candle' with value in config file: True." + "Override process_only_new_candles 'process_only_new_candles' " + "with value in config file: True." ) in caplog.record_tuples From 2ec5a536aadf63006e734b8989fbefa904a95945 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Sep 2018 19:53:49 +0200 Subject: [PATCH 116/175] Fix comment location --- freqtrade/strategy/interface.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 05ede129e..6d865f4bc 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -119,11 +119,12 @@ class IStrategy(ABC): add several TA indicators and buy signal to it :return DataFrame with ticker data and indicator data """ - # Test if seen this pair and last candle before. + dataframe = parse_ticker_dataframe(ticker_history) pair = str(metadata.get('pair')) + # Test if seen this pair and last candle before. # always run if process_only_new_candles is set to true if (not self.process_only_new_candles or self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']): From adfd8c7f5c29f9294b4b8ba9933af69e227d19c0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 2 Sep 2018 14:28:06 +0200 Subject: [PATCH 117/175] Update ccxt from 1.17.216 to 1.17.222 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5cb061855..fbbba7c60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.216 +ccxt==1.17.222 SQLAlchemy==1.2.11 python-telegram-bot==11.0.0 arrow==0.12.1 From 3831f198e9e75e7258ac24a74392cb5f0510f71f Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 2 Sep 2018 14:28:07 +0200 Subject: [PATCH 118/175] Update python-telegram-bot from 11.0.0 to 11.1.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fbbba7c60..38baf322b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ ccxt==1.17.222 SQLAlchemy==1.2.11 -python-telegram-bot==11.0.0 +python-telegram-bot==11.1.0 arrow==0.12.1 cachetools==2.1.0 requests==2.19.1 From e9deb928f6938375c2d3a008f789191c03ce326b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 Sep 2018 19:15:23 +0200 Subject: [PATCH 119/175] Fix bug when exchange result is empty --- freqtrade/exchange/__init__.py | 3 ++- freqtrade/tests/exchange/test_exchange.py | 22 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index fbc84c9b9..f663420e0 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -454,7 +454,8 @@ class Exchange(object): data = sorted(data, key=lambda x: x[0]) # keeping last candle time as last refreshed time of the pair - self._pairs_last_refresh_time[pair] = data[-1][0] // 1000 + if data: + self._pairs_last_refresh_time[pair] = data[-1][0] // 1000 # keeping candles in cache self.klines[pair] = data diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 3c90f425c..de720b3d9 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -635,8 +635,6 @@ async def test__async_get_candle_history(default_conf, mocker, caplog): ] ] - async def async_fetch_ohlcv(pair, timeframe, since): - return tick caplog.set_level(logging.DEBUG) exchange = get_patched_exchange(mocker, default_conf) # Monkey-patch async function @@ -669,6 +667,26 @@ async def test__async_get_candle_history(default_conf, mocker, caplog): (arrow.utcnow().timestamp - 2000) * 1000) +@pytest.mark.asyncio +async def test__async_get_candle_history_empty(default_conf, mocker, caplog): + """ Test empty exchange result """ + tick = [] + + caplog.set_level(logging.DEBUG) + exchange = get_patched_exchange(mocker, default_conf) + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro([]) + + exchange = Exchange(default_conf) + pair = 'ETH/BTC' + res = await exchange._async_get_candle_history(pair, "5m") + assert type(res) is tuple + assert len(res) == 2 + assert res[0] == pair + assert res[1] == tick + assert exchange._api_async.fetch_ohlcv.call_count == 1 + + @pytest.mark.asyncio async def test_async_get_candles_history(default_conf, mocker): tick = [ From 754027efedf2c3a996c4480e50a10d93535b7b9b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 3 Sep 2018 14:28:07 +0200 Subject: [PATCH 120/175] Update ccxt from 1.17.222 to 1.17.223 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 38baf322b..56ff17113 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.222 +ccxt==1.17.223 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From d62f97dc3bc56896109c5fa711fc23bbc86c650a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 4 Sep 2018 14:28:06 +0200 Subject: [PATCH 121/175] Update ccxt from 1.17.223 to 1.17.229 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 56ff17113..062c36e98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.223 +ccxt==1.17.229 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From 27ffce4c3f2c4c7e30f5c0766a427436003c7b01 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 4 Sep 2018 14:28:08 +0200 Subject: [PATCH 122/175] Update pytest-cov from 2.5.1 to 2.6.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 062c36e98..dfcce9ee4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ numpy==1.15.1 TA-Lib==0.4.17 pytest==3.7.4 pytest-mock==1.10.0 -pytest-cov==2.5.1 +pytest-cov==2.6.0 tabulate==0.8.2 coinmarketcap==5.0.3 From a748c0794e4832c7c7f0d7bf6bb501479188afbf Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 5 Sep 2018 14:28:06 +0200 Subject: [PATCH 123/175] Update ccxt from 1.17.229 to 1.17.231 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dfcce9ee4..447478f2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.229 +ccxt==1.17.231 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From 4f583d61c801de09c8778f1512ab69a4598d71e5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 6 Sep 2018 14:28:06 +0200 Subject: [PATCH 124/175] Update ccxt from 1.17.231 to 1.17.233 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 447478f2e..2346f0e25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.231 +ccxt==1.17.233 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From 4e847f26bcd71e911572e35bcb7fa48dcb3d3af2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 6 Sep 2018 20:11:29 +0200 Subject: [PATCH 125/175] explicitly ask for more ressources in hyperopt documentation --- docs/hyperopt.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index f4b69b632..3f568d82e 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -5,6 +5,8 @@ algorithms included in the `scikit-optimize` package to accomplish this. The search will burn all your CPU cores, make your laptop sound like a fighter jet and still take a long time. +*Note:* Hyperopt will crash when used with only 1 CPU Core as found out in [Issue #1133](https://github.com/freqtrade/freqtrade/issues/1133) + ## Table of Contents - [Prepare your Hyperopt](#prepare-hyperopt) - [Configure your Guards and Triggers](#configure-your-guards-and-triggers) From a49a60b4fac013aef5648073cf75bd4ef58c78f4 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 7 Sep 2018 14:28:07 +0200 Subject: [PATCH 126/175] Update ccxt from 1.17.233 to 1.17.240 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2346f0e25..270e8e162 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.233 +ccxt==1.17.240 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From fb4f83b32c0c589a1cc4ec31f7b0a2340817686a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 7 Sep 2018 14:28:09 +0200 Subject: [PATCH 127/175] Update pytest from 3.7.4 to 3.8.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 270e8e162..1b03f6bb2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.1 TA-Lib==0.4.17 -pytest==3.7.4 +pytest==3.8.0 pytest-mock==1.10.0 pytest-cov==2.6.0 tabulate==0.8.2 From e57be10772012d4a3ca5d0b1047e9bb29a3c9be9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 8 Sep 2018 13:01:33 +0200 Subject: [PATCH 128/175] Document ccxt_rate_limit --- docs/configuration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuration.md b/docs/configuration.md index 4674f4b40..010e693d4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -44,6 +44,7 @@ The table below will list all configuration parameters. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. | `exchange.pair_whitelist` | [] | No | List of currency to use by the bot. Can be overrided with `--dynamic-whitelist` param. | `exchange.pair_blacklist` | [] | No | List of currency the bot must avoid. Useful when using `--dynamic-whitelist` param. +| `exchange.ccxt_rate_limit` | True | No | Have CCXT handle Exchange rate limits. Depending on the exchange, having this to false can lead to temporary bans from the exchange. | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. | `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` From 65699f702ecb34c037384411c1c65d1ac1a6843b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 8 Sep 2018 14:27:07 +0200 Subject: [PATCH 129/175] Update ccxt from 1.17.240 to 1.17.242 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1b03f6bb2..e9da93b1d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.240 +ccxt==1.17.242 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From 65ad9cf7410da8f98f46b6f8eb0072d687908dfc Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 9 Sep 2018 14:27:06 +0200 Subject: [PATCH 130/175] Update ccxt from 1.17.242 to 1.17.250 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index feeff061f..d2dd90448 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.242 +ccxt==1.17.250 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From 8aaf174578f16645a813c7f9bf70d188c8ca212d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 10 Sep 2018 14:27:08 +0200 Subject: [PATCH 131/175] Update ccxt from 1.17.250 to 1.17.257 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d2dd90448..07a05c375 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.250 +ccxt==1.17.257 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From 0a29096794989a8749ba3ba53be1e59d52ccc4ad Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 10 Sep 2018 20:19:12 +0200 Subject: [PATCH 132/175] Refactor load_market out of validate_pairs --- freqtrade/exchange/__init__.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index f663420e0..9cd6b861c 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -98,6 +98,7 @@ class Exchange(object): logger.info('Using Exchange "%s"', self.name) + self.markets = self._load_markets() # Check if all pairs are available self.validate_pairs(config['exchange']['pair_whitelist']) @@ -167,6 +168,16 @@ class Exchange(object): logger.warning('Could not load async markets. Reason: %s', e) return + def _load_markets(self) -> List[str]: + """ Initialize markets both sync and async """ + try: + markets = self._api.load_markets() + self._load_async_markets() + return markets + except ccxt.BaseError as e: + logger.warning('Unable to initialize markets. Reason: %s', e) + return [] + def validate_pairs(self, pairs: List[str]) -> None: """ Checks if all given pairs are tradable on the current exchange. @@ -175,11 +186,8 @@ class Exchange(object): :return: None """ - try: - markets = self._api.load_markets() - self._load_async_markets() - except ccxt.BaseError as e: - logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e) + if not self.markets: + logger.warning('Unable to validate pairs (assuming they are correct).') return stake_cur = self._conf['stake_currency'] @@ -189,7 +197,7 @@ class Exchange(object): if not pair.endswith(stake_cur): raise OperationalException( f'Pair {pair} not compatible with stake_currency: {stake_cur}') - if pair not in markets: + if pair not in self.markets: raise OperationalException( f'Pair {pair} is not available at {self.name}') From f954efbd6452832da17ff5a03606bc22bff7dfc0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 10 Sep 2018 20:19:28 +0200 Subject: [PATCH 133/175] Adapt tests to not _load_markets --- freqtrade/tests/conftest.py | 6 +- freqtrade/tests/test_freqtradebot.py | 115 +++++++++++++-------------- 2 files changed, 59 insertions(+), 62 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index af9062cab..9d0396d25 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -4,7 +4,7 @@ import logging from datetime import datetime from functools import reduce from typing import Dict, Optional -from unittest.mock import MagicMock +from unittest.mock import MagicMock, PropertyMock import arrow import pytest @@ -26,8 +26,10 @@ def log_has(line, logs): def patch_exchange(mocker, api_mock=None) -> None: - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value=[])) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value="Bittrex")) + mocker.patch('freqtrade.exchange.Exchange.id', PropertyMock(return_value="bittrex")) if api_mock: mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) else: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 5e982f11a..3bbc01f37 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -140,7 +140,6 @@ def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - # mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) # Test to retrieved BTC sorted on quoteVolume (default) whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC') @@ -175,9 +174,9 @@ def test_refresh_whitelist() -> None: def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_balance=MagicMock(return_value=default_conf['stake_amount'] * 2) ) @@ -193,9 +192,9 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, fee, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5) ) freqtrade = FreqtradeBot(default_conf) @@ -211,6 +210,7 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -253,7 +253,7 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, def test_get_min_pair_stake_amount(mocker, default_conf) -> None: patch_RPCManager(mocker) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + patch_exchange(mocker) freqtrade = FreqtradeBot(default_conf) freqtrade.strategy.stoploss = -0.05 # no pair found @@ -388,9 +388,9 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: def test_create_trade(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -422,9 +422,9 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, markets, mocke def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5), @@ -441,10 +441,10 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=buy_mock, get_fee=fee, @@ -462,10 +462,10 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=buy_mock, get_fee=fee, @@ -483,9 +483,9 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_balance=MagicMock(return_value=default_conf['stake_amount']), @@ -504,9 +504,9 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -527,9 +527,9 @@ def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, marke def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -550,9 +550,9 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: default_conf['dry_run'] = True patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_balance=MagicMock(return_value=20), get_fee=fee, ) @@ -568,9 +568,9 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: def test_process_trade_creation(default_conf, ticker, limit_buy_order, markets, fee, mocker, caplog) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_markets=markets, buy=MagicMock(return_value={'id': limit_buy_order['id']}), @@ -605,9 +605,9 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_markets=markets, buy=MagicMock(side_effect=TemporaryError) @@ -624,9 +624,9 @@ def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> Non def test_process_operational_exception(default_conf, ticker, markets, mocker) -> None: msg_mock = patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_markets=markets, buy=MagicMock(side_effect=OperationalException) @@ -645,9 +645,9 @@ def test_process_operational_exception(default_conf, ticker, markets, mocker) -> def test_process_trade_handling( default_conf, ticker, limit_buy_order, markets, fee, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_markets=markets, buy=MagicMock(return_value={'id': limit_buy_order['id']}), @@ -771,9 +771,9 @@ def test_process_maybe_execute_sell_exception(mocker, default_conf, def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, @@ -814,9 +814,9 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, default_conf.update({'experimental': {'use_sell_signal': True}}) patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -870,9 +870,9 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, default_conf.update({'experimental': {'use_sell_signal': True}}) patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -903,9 +903,9 @@ def test_handle_trade_experimental( caplog.set_level(logging.DEBUG) default_conf.update({'experimental': {'use_sell_signal': True}}) patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -931,9 +931,9 @@ def test_handle_trade_experimental( def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -959,9 +959,9 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fee, mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_order=MagicMock(return_value=limit_buy_order_old), cancel_order=cancel_order_mock, @@ -996,9 +996,9 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_order=MagicMock(return_value=limit_sell_order_old), cancel_order=cancel_order_mock @@ -1032,9 +1032,9 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_order=MagicMock(return_value=limit_buy_order_old_partial), cancel_order=cancel_order_mock @@ -1069,6 +1069,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) cancel_order_mock = MagicMock() mocker.patch.multiple( @@ -1078,7 +1079,6 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) - ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_order=MagicMock(side_effect=requests.exceptions.RequestException('Oh snap')), cancel_order=cancel_order_mock @@ -1111,10 +1111,10 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) - def test_handle_timedout_limit_buy(mocker, default_conf) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) cancel_order_mock = MagicMock() mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), cancel_order=cancel_order_mock ) @@ -1133,10 +1133,10 @@ def test_handle_timedout_limit_buy(mocker, default_conf) -> None: def test_handle_timedout_limit_sell(mocker, default_conf) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) cancel_order_mock = MagicMock() mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), cancel_order=cancel_order_mock ) @@ -1155,9 +1155,9 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None: def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -1174,7 +1174,6 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker_sell_up ) @@ -1201,9 +1200,9 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -1220,7 +1219,6 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, # Decrease the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker_sell_down ) @@ -1249,9 +1247,9 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -1268,7 +1266,6 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker_sell_up ) freqtrade.config = {} @@ -1295,9 +1292,9 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -1314,7 +1311,6 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, # Decrease the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker_sell_down ) @@ -1342,9 +1338,9 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': 0.00002172, 'ask': 0.00002173, @@ -1374,9 +1370,9 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': 0.00002172, 'ask': 0.00002173, @@ -1404,9 +1400,9 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': 0.00000172, 'ask': 0.00000173, @@ -1435,9 +1431,9 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': 0.0000172, 'ask': 0.0000173, @@ -1467,9 +1463,9 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': 0.0000172, 'ask': 0.0000173, @@ -1501,9 +1497,9 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': 0.00000102, 'ask': 0.00000103, @@ -1535,9 +1531,9 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets caplog, mocker) -> None: buy_price = limit_buy_order['price'] patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': buy_price - 0.000001, 'ask': buy_price - 0.000001, @@ -1593,9 +1589,9 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, caplog, mocker, markets) -> None: buy_price = limit_buy_order['price'] patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': buy_price - 0.000001, 'ask': buy_price - 0.000001, @@ -1653,9 +1649,9 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': 0.00000172, 'ask': 0.00000173, @@ -1689,7 +1685,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, caplog, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) patch_RPCManager(mocker) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( pair='LTC/ETH', @@ -1712,7 +1708,7 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) patch_RPCManager(mocker) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) amount = buy_order_fee['amount'] trade = Trade( pair='LTC/ETH', @@ -1735,7 +1731,7 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, mo trades_for_order[0]['fee']['currency'] = 'ETH' patch_RPCManager(mocker) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -1757,7 +1753,7 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock trades_for_order[0]['fee']['cost'] = 0.00094518 patch_RPCManager(mocker) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -1776,7 +1772,7 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, mocker): patch_RPCManager(mocker) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order2) amount = float(sum(x['amount'] for x in trades_for_order2)) trade = Trade( @@ -1801,7 +1797,7 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'} patch_RPCManager(mocker) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[trades_for_order]) amount = float(sum(x['amount'] for x in trades_for_order)) @@ -1827,7 +1823,7 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order limit_buy_order['fee'] = {'cost': 0.004} patch_RPCManager(mocker) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) amount = float(sum(x['amount'] for x in trades_for_order)) trade = Trade( @@ -1849,7 +1845,7 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, trades_for_order[0]['fee'] = {'cost': 0.008} patch_RPCManager(mocker) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -1867,7 +1863,7 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, def test_get_real_amount_open_trade(default_conf, mocker): patch_RPCManager(mocker) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) + patch_exchange(mocker) amount = 12345 trade = Trade( pair='LTC/ETH', @@ -1891,10 +1887,10 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -1927,10 +1923,10 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_o # delta is 100 which is impossible to reach. hence check_depth_of_market will return false default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100 patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -1950,9 +1946,9 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, markets) test if function get_target_bid will return the order book price instead of the ask rate """ + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_markets=markets, get_order_book=order_book_l2 ) @@ -1971,9 +1967,9 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) test if function get_target_bid will return the ask rate (since its value is lower) instead of the order book rate (even if enabled) """ + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_markets=markets, get_order_book=order_book_l2 ) @@ -1992,9 +1988,9 @@ def test_order_book_bid_strategy3(default_conf, mocker, order_book_l2, markets) test if function get_target_bid will return ask rate instead of the order book rate """ + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_markets=markets, get_order_book=order_book_l2 ) @@ -2013,9 +2009,9 @@ def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2, markets) """ test check depth of market """ + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_markets=markets, get_order_book=order_book_l2 ) @@ -2044,7 +2040,6 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, From 51ef13798171edddc51a80eb8cc21febf5b69d02 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 11 Sep 2018 14:27:07 +0200 Subject: [PATCH 134/175] Update ccxt from 1.17.257 to 1.17.271 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 07a05c375..19c6b9db4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.257 +ccxt==1.17.271 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From 14b7fc42fad1f5c047712cad82db26338629d691 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Sep 2018 19:46:18 +0200 Subject: [PATCH 135/175] Change returntype for _load_markets to dict --- freqtrade/exchange/__init__.py | 8 ++++---- freqtrade/tests/conftest.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 9cd6b861c..5eaa2b377 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -168,7 +168,7 @@ class Exchange(object): logger.warning('Could not load async markets. Reason: %s', e) return - def _load_markets(self) -> List[str]: + def _load_markets(self) -> Dict[str, Any]: """ Initialize markets both sync and async """ try: markets = self._api.load_markets() @@ -176,7 +176,7 @@ class Exchange(object): return markets except ccxt.BaseError as e: logger.warning('Unable to initialize markets. Reason: %s', e) - return [] + return {} def validate_pairs(self, pairs: List[str]) -> None: """ @@ -188,7 +188,7 @@ class Exchange(object): if not self.markets: logger.warning('Unable to validate pairs (assuming they are correct).') - return + # return stake_cur = self._conf['stake_currency'] for pair in pairs: @@ -197,7 +197,7 @@ class Exchange(object): if not pair.endswith(stake_cur): raise OperationalException( f'Pair {pair} not compatible with stake_currency: {stake_cur}') - if pair not in self.markets: + if self.markets and pair not in self.markets: raise OperationalException( f'Pair {pair} is not available at {self.name}') diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 9d0396d25..c6eeebbef 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -26,7 +26,7 @@ def log_has(line, logs): def patch_exchange(mocker, api_mock=None) -> None: - mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value=[])) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value="Bittrex")) mocker.patch('freqtrade.exchange.Exchange.id', PropertyMock(return_value="bittrex")) From 674bad2a4f9b18abf71f4ab0e6d21937a3caf077 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Sep 2018 19:46:47 +0200 Subject: [PATCH 136/175] Add and fix tests for load_markets --- freqtrade/tests/exchange/test_exchange.py | 46 ++++++++++++++++------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index de720b3d9..d9d68c3b8 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -183,6 +183,29 @@ def test__load_async_markets(default_conf, mocker, caplog): caplog.record_tuples) +def test__load_markets(default_conf, mocker, caplog): + caplog.set_level(logging.INFO) + api_mock = MagicMock() + mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) + + api_mock.load_markets = MagicMock(return_value={}) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) + + expected_return = {'ETH/BTC': 'available'} + api_mock.load_markets = MagicMock(return_value=expected_return) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + default_conf['exchange']['pair_whitelist'] = ['ETH/BTC'] + ex = Exchange(default_conf) + assert ex.markets == expected_return + + api_mock.load_markets = MagicMock(side_effect=ccxt.BaseError()) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + Exchange(default_conf) + assert log_has('Unable to initialize markets. Reason: ', caplog.record_tuples) + + def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() api_mock.load_markets = MagicMock(return_value={ @@ -199,7 +222,7 @@ def test_validate_pairs(default_conf, mocker): def test_validate_pairs_not_available(default_conf, mocker): api_mock = MagicMock() - api_mock.load_markets = MagicMock(return_value={}) + api_mock.load_markets = MagicMock(return_value={'XRP/BTC': 'inactive'}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) @@ -234,11 +257,9 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available at Binance'): Exchange(default_conf) - api_mock.load_markets = MagicMock(side_effect=ccxt.BaseError()) - - mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) Exchange(default_conf) - assert log_has('Unable to validate pairs (assuming they are correct). Reason: ', + assert log_has('Unable to validate pairs (assuming they are correct).', caplog.record_tuples) @@ -270,7 +291,7 @@ def test_validate_timeframes(default_conf, mocker): type(api_mock).timeframes = timeframes mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) Exchange(default_conf) @@ -286,7 +307,7 @@ def test_validate_timeframes_failed(default_conf, mocker): type(api_mock).timeframes = timeframes mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) with pytest.raises(OperationalException, match=r'Invalid ticker 3m, this Exchange supports.*'): Exchange(default_conf) @@ -303,7 +324,7 @@ def test_validate_timeframes_not_in_config(default_conf, mocker): type(api_mock).timeframes = timeframes mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) Exchange(default_conf) @@ -940,8 +961,7 @@ def test_get_order(default_conf, mocker): def test_name(default_conf, mocker): - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', - side_effect=lambda s: True) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) default_conf['exchange']['name'] = 'binance' exchange = Exchange(default_conf) @@ -949,16 +969,14 @@ def test_name(default_conf, mocker): def test_id(default_conf, mocker): - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', - side_effect=lambda s: True) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) default_conf['exchange']['name'] = 'binance' exchange = Exchange(default_conf) assert exchange.id == 'binance' def test_get_pair_detail_url(default_conf, mocker, caplog): - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', - side_effect=lambda s: True) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) default_conf['exchange']['name'] = 'binance' exchange = Exchange(default_conf) From c429eae6e42c906312fb3e35132b5557e7f8be2b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Sep 2018 19:59:01 +0200 Subject: [PATCH 137/175] Adjust remaining tests to _load_markets refactoring --- freqtrade/tests/rpc/test_rpc.py | 27 ++++++++++------------- freqtrade/tests/rpc/test_rpc_telegram.py | 28 +++++++++++------------- freqtrade/tests/test_freqtradebot.py | 9 ++++---- 3 files changed, 30 insertions(+), 34 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index c17ab6b2f..efc136777 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -13,7 +13,7 @@ from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException from freqtrade.state import State from freqtrade.tests.test_freqtradebot import patch_get_signal -from freqtrade.tests.conftest import patch_coinmarketcap +from freqtrade.tests.conftest import patch_coinmarketcap, patch_exchange # Functions for recurrent object patching @@ -30,7 +30,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), + _load_markets=MagicMock(return_value={}), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -68,10 +68,10 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: patch_coinmarketcap(mocker) + patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -99,10 +99,10 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: def test_rpc_daily_profit(default_conf, update, ticker, fee, limit_buy_order, limit_sell_order, markets, mocker) -> None: patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -152,11 +152,11 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, ticker=MagicMock(return_value={'price_usd': 15000.0}), ) patch_coinmarketcap(mocker) + patch_exchange(mocker) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -182,7 +182,6 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # Update the ticker with a market going up mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker_sell_up ) trade.update(limit_sell_order) @@ -197,7 +196,6 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # Update the ticker with a market going up mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker_sell_up ) trade.update(limit_sell_order) @@ -223,6 +221,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # trade.open_rate (it is set to None) def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, ticker_sell_up, limit_buy_order, limit_sell_order): + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), @@ -231,7 +230,6 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -252,7 +250,6 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, # Update the ticker with a market going up mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker_sell_up, get_fee=fee ) @@ -298,11 +295,11 @@ def test_rpc_balance_handle(default_conf, mocker): ticker=MagicMock(return_value={'price_usd': 15000.0}), ) patch_coinmarketcap(mocker) + patch_exchange(mocker) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_balances=MagicMock(return_value=mock_balance), get_ticker=MagicMock(side_effect=TemporaryError('Could not load ticker due to xxx')) ) @@ -328,10 +325,10 @@ def test_rpc_balance_handle(default_conf, mocker): def test_rpc_start(mocker, default_conf) -> None: patch_coinmarketcap(mocker) + patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=MagicMock() ) @@ -351,10 +348,10 @@ def test_rpc_start(mocker, default_conf) -> None: def test_rpc_stop(mocker, default_conf) -> None: patch_coinmarketcap(mocker) + patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=MagicMock() ) @@ -375,12 +372,12 @@ def test_rpc_stop(mocker, default_conf) -> None: def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: patch_coinmarketcap(mocker) + patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) cancel_order_mock = MagicMock() mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, cancel_order=cancel_order_mock, get_order=MagicMock( @@ -476,10 +473,10 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: def test_performance_handle(default_conf, ticker, limit_buy_order, fee, limit_sell_order, markets, mocker) -> None: patch_coinmarketcap(mocker) + patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_balances=MagicMock(return_value=ticker), get_ticker=ticker, get_fee=fee, @@ -512,10 +509,10 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, def test_rpc_count(mocker, default_conf, ticker, fee, markets) -> None: patch_coinmarketcap(mocker) + patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_balances=MagicMock(return_value=ticker), get_ticker=ticker, get_fee=fee, diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 4d2b9cda2..182c1d2e7 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -177,10 +177,9 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: default_conf['telegram']['chat_id'] = 123 patch_coinmarketcap(mocker) - + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_pair_detail_url=MagicMock(), get_fee=fee, @@ -228,9 +227,9 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> None: patch_coinmarketcap(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -273,9 +272,9 @@ def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> No def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) -> None: patch_coinmarketcap(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_fee=fee, @@ -324,13 +323,13 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, limit_sell_order, markets, mocker) -> None: patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + patch_exchange(mocker) mocker.patch( 'freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0 ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -395,9 +394,9 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker ) msg_mock = MagicMock() @@ -431,10 +430,10 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, limit_buy_order, limit_sell_order, markets, mocker) -> None: patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + patch_exchange(mocker) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -678,7 +677,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), + _load_markets=MagicMock(return_value={}), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -727,7 +726,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), + _load_markets=MagicMock(return_value={}), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -743,7 +742,6 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, # Decrease the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker_sell_down ) @@ -775,13 +773,13 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker) -> None: patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + patch_exchange(mocker) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.get_pair_detail_url', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -827,7 +825,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + patch_exchange(mocker) freqtradebot = FreqtradeBot(default_conf) patch_get_signal(freqtradebot, (True, False)) @@ -860,6 +858,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: def test_performance_handle(default_conf, update, ticker, fee, limit_buy_order, limit_sell_order, markets, mocker) -> None: patch_coinmarketcap(mocker) + patch_exchange(mocker) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -868,7 +867,6 @@ def test_performance_handle(default_conf, update, ticker, fee, ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -899,13 +897,13 @@ def test_performance_handle(default_conf, update, ticker, fee, def test_performance_handle_invalid(default_conf, update, mocker) -> None: patch_coinmarketcap(mocker) + patch_exchange(mocker) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) freqtradebot = FreqtradeBot(default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -919,6 +917,7 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None: def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> None: patch_coinmarketcap(mocker) + patch_exchange(mocker) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -927,7 +926,6 @@ def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> Non ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_markets=markets diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 3bbc01f37..0b17a2475 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1155,9 +1155,9 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None: def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', + _load_markets=MagicMock(return_value={}), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -1200,9 +1200,9 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', + _load_markets=MagicMock(return_value={}), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -1247,9 +1247,9 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', + _load_markets=MagicMock(return_value={}), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -1292,9 +1292,9 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', + _load_markets=MagicMock(return_value={}), get_ticker=ticker, get_fee=fee, get_markets=markets @@ -2038,6 +2038,7 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order default_conf['ask_strategy']['order_book_max'] = 2 default_conf['telegram']['enabled'] = False patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=MagicMock(return_value={ From 241b23e5d856599e7d0381b2693acafeaed4b5d9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 12 Sep 2018 14:28:06 +0200 Subject: [PATCH 138/175] Update ccxt from 1.17.271 to 1.17.276 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 19c6b9db4..720894ec4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.271 +ccxt==1.17.276 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From 91c0e3640fdd76583d42c6dfc9cf3ab075b2e162 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 13 Sep 2018 14:29:06 +0200 Subject: [PATCH 139/175] Update ccxt from 1.17.276 to 1.17.283 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 720894ec4..04aab1954 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.276 +ccxt==1.17.283 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From f5ba34addfc90206edf160e8710c21ba81d23305 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 14 Sep 2018 14:28:05 +0200 Subject: [PATCH 140/175] Update ccxt from 1.17.283 to 1.17.291 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 04aab1954..b9e5ff08e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.283 +ccxt==1.17.291 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From f4d26961c87b6942b4afa2faf000522cbec8fc49 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 15 Sep 2018 14:28:05 +0200 Subject: [PATCH 141/175] Update ccxt from 1.17.291 to 1.17.294 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b9e5ff08e..cd33410ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.291 +ccxt==1.17.294 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From 9685c09c1a58ff36bf1e836a15fcdbb3ca049bc9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Sep 2018 20:28:36 +0200 Subject: [PATCH 142/175] Add offset to "get_trades_for_order" --- freqtrade/exchange/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index f663420e0..e3d51a16f 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -591,7 +591,8 @@ class Exchange(object): if not self.exchange_has('fetchMyTrades'): return [] try: - my_trades = self._api.fetch_my_trades(pair, since.timestamp()) + # Allow 5s offset to catch slight time offsets (discovered in #1185) + my_trades = self._api.fetch_my_trades(pair, since.timestamp() - 5) matched_trades = [trade for trade in my_trades if trade['order'] == order_id] return matched_trades From 51b3eb78d78dc2cec70c34e10b2d4e6efd53ba9e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Sep 2018 20:38:09 +0200 Subject: [PATCH 143/175] Add section about about clock accuracy to readme.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 02b870209..2b44c99ad 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,10 @@ to understand the requirements before sending your pull-requests. ## Requirements +### Uptodate clock + +The clock must be uptodate, ideally syncronized to a NTP server rather frequently. Drifting time can lead to losses. + ### Min hardware required To run this bot we recommend you a cloud instance with a minimum of: From 200dfa75759b2d3e439315c5157c6fc4aa2c9038 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Sep 2018 11:22:15 +0200 Subject: [PATCH 144/175] Wording for readme.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2b44c99ad..65895ac29 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ to understand the requirements before sending your pull-requests. ### Uptodate clock -The clock must be uptodate, ideally syncronized to a NTP server rather frequently. Drifting time can lead to losses. +The clock must be accurate, syncronized to a NTP server very frequently to avoid problems with communication to the exchanges. ### Min hardware required From 30ae5829f5c200b234df17a3bb8d58fad948ed68 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Sep 2018 11:24:49 +0200 Subject: [PATCH 145/175] Fix SED command for macos Mac uses the bsd version, where -i without backup is not allowed. --- docs/installation.md | 2 +- install_ta-lib.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 0fecfcf78..1ceda6b1c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -267,7 +267,7 @@ Official webpage: https://mrjbq7.github.io/ta-lib/install.html wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar xvzf ta-lib-0.4.0-src.tar.gz cd ta-lib -sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h +sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h ./configure --prefix=/usr make make install diff --git a/install_ta-lib.sh b/install_ta-lib.sh index 18e7b8bbb..d8ae2eeaa 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -1,6 +1,6 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib && sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd .. + cd ta-lib && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd .. else echo "TA-lib already installed, skipping download and build." cd ta-lib && sudo make install && cd .. From 14961e2e38c2bcd10d366e62a8738f842512bea2 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 16 Sep 2018 14:28:06 +0200 Subject: [PATCH 146/175] Update ccxt from 1.17.294 to 1.17.300 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cd33410ea..19e19c13b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.294 +ccxt==1.17.300 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From 6aa18bddc9cd012bf25f46028e59eae116f68463 Mon Sep 17 00:00:00 2001 From: 0xflotus <0xflotus@gmail.com> Date: Sun, 16 Sep 2018 17:34:01 +0200 Subject: [PATCH 147/175] fixed being --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 02b870209..1633bbf13 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ The project is currently setup in two main branches: - `develop` - This branch has often new features, but might also cause breaking changes. - `master` - This branch contains the latest stable release. The bot 'should' be stable on this branch, and is generally well tested. -- `feat/*` - These are feature branches, which are beeing worked on heavily. Please don't use these unless you want to test a specific feature. +- `feat/*` - These are feature branches, which are being worked on heavily. Please don't use these unless you want to test a specific feature. ## A note on Binance From 0a4b2f19e375a3f2306ccdffd03b31bb402a458d Mon Sep 17 00:00:00 2001 From: gaojin Date: Mon, 17 Sep 2018 10:37:25 +0800 Subject: [PATCH 148/175] use --no-cache-dir for docker build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit use --no-cache can save about 90M ``` ➜ freqtrade git:(develop) ✗ docker images freq REPOSITORY TAG IMAGE ID CREATED SIZE freq latest b15db8341067 7 minutes ago 800MB ➜ freqtrade git:(develop) ✗ docker images freq_nocache REPOSITORY TAG IMAGE ID CREATED SIZE freq_nocache latest e5731f28ac54 20 seconds ago 709MB ``` --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5d1b44f8c..aa9071024 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,11 +15,11 @@ RUN mkdir /freqtrade WORKDIR /freqtrade # Install dependencies -COPY requirements.txt /freqtrade/ -RUN pip install numpy \ - && pip install -r requirements.txt +COPY requirements.txt /freqtrade/ +RUN pip install numpy --no-cache-dir \ + && pip install -r requirements.txt --no-cache-dir # Install and execute COPY . /freqtrade/ -RUN pip install -e . +RUN pip install -e . --no-cache-dir ENTRYPOINT ["freqtrade"] From 9b83a0922490ec369047ec371b6c7ca426d4e8f5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 17 Sep 2018 14:28:06 +0200 Subject: [PATCH 149/175] Update ccxt from 1.17.300 to 1.17.305 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 19e19c13b..07c305322 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.300 +ccxt==1.17.305 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From 14e21765f207a29ac85900528114ac8449f0a00b Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 17 Sep 2018 19:44:40 +0200 Subject: [PATCH 150/175] Fix missing column to load current backtesting export files --- scripts/plot_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 0f0a3d4cb..68713f296 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -73,7 +73,7 @@ def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFram file = Path(args.exportfilename) # must align with columns in backtest.py columns = ["pair", "profit", "opents", "closets", "index", "duration", - "open_rate", "close_rate", "open_at_end"] + "open_rate", "close_rate", "open_at_end", "sell_reason"] with file.open() as f: data = json.load(f) trades = pd.DataFrame(data, columns=columns) From 176bae2d598dac5d5d42c2d35b93d5915efd56e1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 17 Sep 2018 19:57:47 +0200 Subject: [PATCH 151/175] Set default-db url in configuration, not arguments * Fixes a bug in plot_dataframe.py (#1217) * db_url is eventually overwritten here anyway. --- freqtrade/arguments.py | 1 - freqtrade/configuration.py | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 501c1784f..bb571b4ea 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -119,7 +119,6 @@ class Arguments(object): help='Override trades database URL, this is useful if dry_run is enabled' ' or in custom deployments (default: %(default)s)', dest='db_url', - default=constants.DEFAULT_DB_PROD_URL, type=str, metavar='PATH', ) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 3da432b1d..4e1137f33 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -110,9 +110,12 @@ class Configuration(object): '(not applicable with Backtesting and Hyperopt)' ) - if self.args.db_url != constants.DEFAULT_DB_PROD_URL: + if self.args.db_url and self.args.db_url != constants.DEFAULT_DB_PROD_URL: config.update({'db_url': self.args.db_url}) logger.info('Parameter --db-url detected ...') + else: + # Set default here + config.update({'db_url': constants.DEFAULT_DB_PROD_URL}) if config.get('dry_run', False): logger.info('Dry run is enabled') From f04e4f21230c89c055deaa33e2ee676cab5bd7d3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 17 Sep 2018 20:49:41 +0200 Subject: [PATCH 152/175] Fix trailing whitespace --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aa9071024..2506665ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN mkdir /freqtrade WORKDIR /freqtrade # Install dependencies -COPY requirements.txt /freqtrade/ +COPY requirements.txt /freqtrade/ RUN pip install numpy --no-cache-dir \ && pip install -r requirements.txt --no-cache-dir From a5d4de803710d9357e69d2646e871fcf92369680 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 18 Sep 2018 14:28:06 +0200 Subject: [PATCH 153/175] Update ccxt from 1.17.305 to 1.17.311 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 07c305322..2fa1ca5a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.305 +ccxt==1.17.311 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From 2c5b6aca91b156c9200a2afda8aaf69356c2e2e9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 19 Sep 2018 14:28:06 +0200 Subject: [PATCH 154/175] Update ccxt from 1.17.311 to 1.17.316 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2fa1ca5a5..149143d9c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.311 +ccxt==1.17.316 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From 4d5e368c2ead3813877e761c66a54b400c8bb329 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Sep 2018 19:40:32 +0200 Subject: [PATCH 155/175] Remove direct call to pytest fixture to elliminate pytest warning --- freqtrade/tests/test_acl_pair.py | 47 ++++++++++++++++---------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 535684b22..38df3cb38 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -2,38 +2,39 @@ from unittest.mock import MagicMock -import freqtrade.tests.conftest as tt # test tools +from freqtrade.tests.conftest import get_patched_freqtradebot + +import pytest # whitelist, blacklist, filtering, all of that will # eventually become some rules to run on a generic ACL engine # perhaps try to anticipate that by using some python package -def whitelist_conf(): - config = tt.default_conf() - config['stake_currency'] = 'BTC' - config['exchange']['pair_whitelist'] = [ +@pytest.fixture(scope="function") +def whitelist_conf(default_conf): + default_conf['stake_currency'] = 'BTC' + default_conf['exchange']['pair_whitelist'] = [ 'ETH/BTC', 'TKN/BTC', 'TRST/BTC', 'SWT/BTC', 'BCC/BTC' ] - config['exchange']['pair_blacklist'] = [ + default_conf['exchange']['pair_blacklist'] = [ 'BLK/BTC' ] - return config + return default_conf -def test_refresh_market_pair_not_in_whitelist(mocker, markets): - conf = whitelist_conf() +def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): - freqtradebot = tt.get_patched_freqtradebot(mocker, conf) + freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) refreshedwhitelist = freqtradebot._refresh_whitelist( - conf['exchange']['pair_whitelist'] + ['XXX/BTC'] + whitelist_conf['exchange']['pair_whitelist'] + ['XXX/BTC'] ) # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] @@ -41,12 +42,12 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets): assert whitelist == refreshedwhitelist -def test_refresh_whitelist(mocker, markets): - conf = whitelist_conf() - freqtradebot = tt.get_patched_freqtradebot(mocker, conf) +def test_refresh_whitelist(mocker, markets, whitelist_conf): + freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - refreshedwhitelist = freqtradebot._refresh_whitelist(conf['exchange']['pair_whitelist']) + refreshedwhitelist = freqtradebot._refresh_whitelist( + whitelist_conf['exchange']['pair_whitelist']) # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] @@ -54,9 +55,8 @@ def test_refresh_whitelist(mocker, markets): assert whitelist == refreshedwhitelist -def test_refresh_whitelist_dynamic(mocker, markets, tickers): - conf = whitelist_conf() - freqtradebot = tt.get_patched_freqtradebot(mocker, conf) +def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): + freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_markets=markets, @@ -68,21 +68,20 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers): whitelist = ['ETH/BTC', 'TKN/BTC'] refreshedwhitelist = freqtradebot._refresh_whitelist( - freqtradebot._gen_pair_whitelist(conf['stake_currency']) + freqtradebot._gen_pair_whitelist(whitelist_conf['stake_currency']) ) assert whitelist == refreshedwhitelist -def test_refresh_whitelist_dynamic_empty(mocker, markets_empty): - conf = whitelist_conf() - freqtradebot = tt.get_patched_freqtradebot(mocker, conf) +def test_refresh_whitelist_dynamic_empty(mocker, markets_empty, whitelist_conf): + freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets_empty) # argument: use the whitelist dynamically by exchange-volume whitelist = [] - conf['exchange']['pair_whitelist'] = [] + whitelist_conf['exchange']['pair_whitelist'] = [] freqtradebot._refresh_whitelist(whitelist) - pairslist = conf['exchange']['pair_whitelist'] + pairslist = whitelist_conf['exchange']['pair_whitelist'] assert set(whitelist) == set(pairslist) From 0aa8557c03f528a2a6e178f9129d106a968e6e16 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 20 Sep 2018 14:28:08 +0200 Subject: [PATCH 156/175] Update ccxt from 1.17.316 to 1.17.324 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 149143d9c..d9f8f39f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.316 +ccxt==1.17.324 SQLAlchemy==1.2.11 python-telegram-bot==11.1.0 arrow==0.12.1 From 53c0f01bef9d4cbc54c6a086c52682f6269ca534 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 20 Sep 2018 14:28:10 +0200 Subject: [PATCH 157/175] Update sqlalchemy from 1.2.11 to 1.2.12 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d9f8f39f1..676f8b0cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ccxt==1.17.324 -SQLAlchemy==1.2.11 +SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 cachetools==2.1.0 From 567211e9f9e14f5722fea74686ca45d09c5fb759 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Sep 2018 20:35:26 +0200 Subject: [PATCH 158/175] don't print "NAN" lines in "left_open_trades" --- freqtrade/optimize/backtesting.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d0b70afc7..0983ec677 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -108,7 +108,8 @@ class Backtesting(object): return min(timeframe, key=operator.itemgetter(0))[0], \ max(timeframe, key=operator.itemgetter(1))[1] - def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str: + def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame, + skip_nan: bool = False) -> str: """ Generates and returns a text table for the given backtest data and the results dataframe :return: pretty printed table with tabulate as str @@ -121,6 +122,9 @@ class Backtesting(object): 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] for pair in data: result = results[results.pair == pair] + if skip_nan and result.profit_abs.isnull().all(): + continue + tabular_data.append([ pair, len(result.index), @@ -404,7 +408,7 @@ class Backtesting(object): print(self._generate_text_table_sell_reason(data, results)) print(' LEFT OPEN TRADES REPORT '.center(119, '=')) - print(self._generate_text_table(data, results.loc[results.open_at_end])) + print(self._generate_text_table(data, results.loc[results.open_at_end], True)) print() if len(all_results) > 1: # Print Strategy summary table From 8e659af580b5bfebd57d56b061cc7ff1791c9a77 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 21 Sep 2018 14:28:07 +0200 Subject: [PATCH 159/175] Update ccxt from 1.17.324 to 1.17.327 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 676f8b0cc..6c88f3a19 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.324 +ccxt==1.17.327 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 54b714ba3f1995fc9c4534ef348d223a3ac6a0c4 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 22 Sep 2018 14:28:05 +0200 Subject: [PATCH 160/175] Update ccxt from 1.17.327 to 1.17.335 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6c88f3a19..9709f22f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.327 +ccxt==1.17.335 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 0e168159c163f12987849ace3f8d7d60600ab74b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 23 Sep 2018 14:28:06 +0200 Subject: [PATCH 161/175] Update ccxt from 1.17.335 to 1.17.341 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9709f22f1..ddde2a2e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.335 +ccxt==1.17.341 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 12e6287875473388f4265a0576bf656f17f30b50 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 23 Sep 2018 14:28:08 +0200 Subject: [PATCH 162/175] Update numpy from 1.15.1 to 1.15.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ddde2a2e0..e19d2b8f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ pandas==0.23.4 scikit-learn==0.19.2 scipy==1.1.0 jsonschema==2.6.0 -numpy==1.15.1 +numpy==1.15.2 TA-Lib==0.4.17 pytest==3.8.0 pytest-mock==1.10.0 From 6116c27aa94b1909bf628449d14e2986e6a7a3f8 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 23 Sep 2018 14:28:09 +0200 Subject: [PATCH 163/175] Update pytest from 3.8.0 to 3.8.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e19d2b8f2..8c7f68b86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.2 TA-Lib==0.4.17 -pytest==3.8.0 +pytest==3.8.1 pytest-mock==1.10.0 pytest-asyncio==0.9.0 pytest-cov==2.6.0 From d13e87d7a49dca6f52913298fb35d10aa902fa65 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 25 Sep 2018 14:28:07 +0200 Subject: [PATCH 164/175] Update ccxt from 1.17.341 to 1.17.350 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8c7f68b86..74b3df91d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.341 +ccxt==1.17.350 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 88ccdc03660ffeac25e033a07bce27807cd07e0e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Sep 2018 20:45:01 +0200 Subject: [PATCH 165/175] Fix exception when order cannot be found --- freqtrade/freqtradebot.py | 4 +-- freqtrade/tests/test_freqtradebot.py | 38 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5c943ba3d..fa803bda7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -10,7 +10,7 @@ from datetime import datetime from typing import Any, Callable, Dict, List, Optional import arrow -import requests +from requests.exceptions import RequestException from cachetools import TTLCache, cached @@ -646,7 +646,7 @@ class FreqtradeBot(object): if not trade.open_order_id: continue order = self.exchange.get_order(trade.open_order_id, trade.pair) - except requests.exceptions.RequestException: + except (RequestException, DependencyException): logger.info( 'Cannot query order for %s due to %s', trade, diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 5e982f11a..1392d4091 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -993,6 +993,44 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe assert nb_trades == 0 +def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_order_old, + fee, mocker) -> None: + rpc_mock = patch_RPCManager(mocker) + cancel_order_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_order=MagicMock(side_effect=DependencyException), + cancel_order=cancel_order_mock, + get_fee=fee + ) + freqtrade = FreqtradeBot(default_conf) + + trade_buy = Trade( + pair='ETH/BTC', + open_rate=0.00001099, + exchange='bittrex', + open_order_id='123456789', + amount=90.99181073, + fee_open=0.0, + fee_close=0.0, + stake_amount=1, + open_date=arrow.utcnow().shift(minutes=-601).datetime, + is_open=True + ) + + Trade.session.add(trade_buy) + + # check it does cancel buy orders over the time limit + freqtrade.check_handle_timedout() + assert cancel_order_mock.call_count == 0 + assert rpc_mock.call_count == 0 + trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() + nb_trades = len(trades) + assert nb_trades == 1 + + def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() From f790f953191239f7b477576686a2ffea43ec91d6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 26 Sep 2018 14:28:07 +0200 Subject: [PATCH 166/175] Update ccxt from 1.17.350 to 1.17.351 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 74b3df91d..59d06aca1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.350 +ccxt==1.17.351 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 6e66763e5f041478e9b4a1196d9fbea0dc28893f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 Sep 2018 19:23:55 +0200 Subject: [PATCH 167/175] Only load strategy once during backtesting --- freqtrade/optimize/backtesting.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0983ec677..cd822023f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -75,8 +75,6 @@ class Backtesting(object): else: # only one strategy - strat = StrategyResolver(self.config).strategy - self.strategylist.append(StrategyResolver(self.config).strategy) # Load one strategy self._set_strategy(self.strategylist[0]) From f4585a2745b11fb239d98b47939095a31f9bfd2f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Sep 2018 13:35:48 +0200 Subject: [PATCH 168/175] Patch exchange to not cause network delays during tests --- freqtrade/tests/test_freqtradebot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index e047cd5d1..cad2f654d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -997,6 +997,7 @@ def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_ord fee, mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), From 334e7553e1afbb664575f14c3cf3fd23a12f597d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Sep 2018 13:46:38 +0200 Subject: [PATCH 169/175] Fix hyperopt not working after update of scikit-learn to 0.20.0 --- freqtrade/strategy/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index 38a110bd7..b29e26ef9 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -3,7 +3,8 @@ import sys from copy import deepcopy from freqtrade.strategy.interface import IStrategy - +# Import Default-Strategy to have hyperopt correctly resolve +from freqtrade.strategy.default_strategy import DefaultStrategy # noqa: F401 logger = logging.getLogger(__name__) From 1b290ffb5d70b1ef6c8e0a70610245d92d880122 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Sep 2018 13:49:38 +0200 Subject: [PATCH 170/175] Update hyperopt to show errors if non-supported variables are used --- freqtrade/optimize/hyperopt.py | 10 ++++++++- freqtrade/tests/optimize/test_hyperopt.py | 25 +++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 086cad5aa..9dfaa8ef3 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -41,6 +41,14 @@ class Hyperopt(Backtesting): hyperopt.start() """ def __init__(self, config: Dict[str, Any]) -> None: + if config.get('strategy') and config.get('strategy') != 'DefaultStrategy': + logger.error("Please don't use --strategy for hyperopt.") + logger.error( + "Read the documentation at " + "https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md " + "to understand how to configure hyperopt.") + raise ValueError("--strategy configured but not supported for hyperopt") + super().__init__(config) # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days @@ -152,7 +160,7 @@ class Hyperopt(Backtesting): @staticmethod def generate_roi_table(params: Dict) -> Dict[int, float]: """ - Generate the ROI table thqt will be used by Hyperopt + Generate the ROI table that will be used by Hyperopt """ roi_table = {} roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 65a3c2fdb..2035e23df 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -65,6 +65,31 @@ def test_start(mocker, default_conf, caplog) -> None: assert start_mock.call_count == 1 +def test_start_failure(mocker, default_conf, caplog) -> None: + start_mock = MagicMock() + mocker.patch( + 'freqtrade.configuration.Configuration._load_config_file', + lambda *args, **kwargs: default_conf + ) + mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) + patch_exchange(mocker) + + args = [ + '--config', 'config.json', + '--strategy', 'TestStrategy', + 'hyperopt', + '--epochs', '5' + ] + args = get_args(args) + StrategyResolver({'strategy': 'DefaultStrategy'}) + with pytest.raises(ValueError): + start(args) + assert log_has( + "Please don't use --strategy for hyperopt.", + caplog.record_tuples + ) + + def test_loss_calculation_prefer_correct_trade_count(hyperopt) -> None: StrategyResolver({'strategy': 'DefaultStrategy'}) From 36e9abc8410f67608f38dac850b9d15755cc5437 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Sep 2018 13:50:02 +0200 Subject: [PATCH 171/175] Manually update scikit-learn to 0.20.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 59d06aca1..4150dc5bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ requests==2.19.1 urllib3==1.22 wrapt==1.10.11 pandas==0.23.4 -scikit-learn==0.19.2 +scikit-learn==0.20.0 scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.2 From 84622dc84b0368bfb1420f87ffcddad7a6807feb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Sep 2018 14:23:53 +0200 Subject: [PATCH 172/175] Move test for strategy out of constructor --- freqtrade/optimize/hyperopt.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 9dfaa8ef3..4a239ab28 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -41,14 +41,6 @@ class Hyperopt(Backtesting): hyperopt.start() """ def __init__(self, config: Dict[str, Any]) -> None: - if config.get('strategy') and config.get('strategy') != 'DefaultStrategy': - logger.error("Please don't use --strategy for hyperopt.") - logger.error( - "Read the documentation at " - "https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md " - "to understand how to configure hyperopt.") - raise ValueError("--strategy configured but not supported for hyperopt") - super().__init__(config) # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days @@ -410,6 +402,13 @@ def start(args: Namespace) -> None: config['exchange']['key'] = '' config['exchange']['secret'] = '' + if config.get('strategy') and config.get('strategy') != 'DefaultStrategy': + logger.error("Please don't use --strategy for hyperopt.") + logger.error( + "Read the documentation at " + "https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md " + "to understand how to configure hyperopt.") + raise ValueError("--strategy configured but not supported for hyperopt") # Initialize backtesting object hyperopt = Hyperopt(config) hyperopt.start() From 05adebb5362062a33aae9e546ce0e46e0a6c29c2 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 30 Sep 2018 14:28:06 +0200 Subject: [PATCH 173/175] Update ccxt from 1.17.351 to 1.17.360 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 59d06aca1..5b4a60fac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.351 +ccxt==1.17.360 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 9d70d25064dcfb2e44b9e178ef892364e3acc983 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 30 Sep 2018 14:28:07 +0200 Subject: [PATCH 174/175] Update scikit-learn from 0.19.2 to 0.20.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5b4a60fac..0045e7458 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ requests==2.19.1 urllib3==1.22 wrapt==1.10.11 pandas==0.23.4 -scikit-learn==0.19.2 +scikit-learn==0.20.0 scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.2 From d0c7b7c5821f29e324d7365cc60bfc5718e555a3 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 1 Oct 2018 14:29:06 +0200 Subject: [PATCH 175/175] Update ccxt from 1.17.360 to 1.17.363 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0045e7458..fc5d936b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.360 +ccxt==1.17.363 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1