From 9685c09c1a58ff36bf1e836a15fcdbb3ca049bc9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Sep 2018 20:28:36 +0200 Subject: [PATCH 1/9] Add offset to "get_trades_for_order" --- freqtrade/exchange/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index f663420e0..e3d51a16f 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -591,7 +591,8 @@ class Exchange(object): if not self.exchange_has('fetchMyTrades'): return [] try: - my_trades = self._api.fetch_my_trades(pair, since.timestamp()) + # Allow 5s offset to catch slight time offsets (discovered in #1185) + my_trades = self._api.fetch_my_trades(pair, since.timestamp() - 5) matched_trades = [trade for trade in my_trades if trade['order'] == order_id] return matched_trades From 51b3eb78d78dc2cec70c34e10b2d4e6efd53ba9e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Sep 2018 20:38:09 +0200 Subject: [PATCH 2/9] Add section about about clock accuracy to readme.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 02b870209..2b44c99ad 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,10 @@ to understand the requirements before sending your pull-requests. ## Requirements +### Uptodate clock + +The clock must be uptodate, ideally syncronized to a NTP server rather frequently. Drifting time can lead to losses. + ### Min hardware required To run this bot we recommend you a cloud instance with a minimum of: From 200dfa75759b2d3e439315c5157c6fc4aa2c9038 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Sep 2018 11:22:15 +0200 Subject: [PATCH 3/9] Wording for readme.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2b44c99ad..65895ac29 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ to understand the requirements before sending your pull-requests. ### Uptodate clock -The clock must be uptodate, ideally syncronized to a NTP server rather frequently. Drifting time can lead to losses. +The clock must be accurate, syncronized to a NTP server very frequently to avoid problems with communication to the exchanges. ### Min hardware required From 30ae5829f5c200b234df17a3bb8d58fad948ed68 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Sep 2018 11:24:49 +0200 Subject: [PATCH 4/9] Fix SED command for macos Mac uses the bsd version, where -i without backup is not allowed. --- docs/installation.md | 2 +- install_ta-lib.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 0fecfcf78..1ceda6b1c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -267,7 +267,7 @@ Official webpage: https://mrjbq7.github.io/ta-lib/install.html wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar xvzf ta-lib-0.4.0-src.tar.gz cd ta-lib -sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h +sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h ./configure --prefix=/usr make make install diff --git a/install_ta-lib.sh b/install_ta-lib.sh index 18e7b8bbb..d8ae2eeaa 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -1,6 +1,6 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib && sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd .. + cd ta-lib && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd .. else echo "TA-lib already installed, skipping download and build." cd ta-lib && sudo make install && cd .. From 14e21765f207a29ac85900528114ac8449f0a00b Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 17 Sep 2018 19:44:40 +0200 Subject: [PATCH 5/9] Fix missing column to load current backtesting export files --- scripts/plot_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 0f0a3d4cb..68713f296 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -73,7 +73,7 @@ def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFram file = Path(args.exportfilename) # must align with columns in backtest.py columns = ["pair", "profit", "opents", "closets", "index", "duration", - "open_rate", "close_rate", "open_at_end"] + "open_rate", "close_rate", "open_at_end", "sell_reason"] with file.open() as f: data = json.load(f) trades = pd.DataFrame(data, columns=columns) From 176bae2d598dac5d5d42c2d35b93d5915efd56e1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 17 Sep 2018 19:57:47 +0200 Subject: [PATCH 6/9] Set default-db url in configuration, not arguments * Fixes a bug in plot_dataframe.py (#1217) * db_url is eventually overwritten here anyway. --- freqtrade/arguments.py | 1 - freqtrade/configuration.py | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 501c1784f..bb571b4ea 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -119,7 +119,6 @@ class Arguments(object): help='Override trades database URL, this is useful if dry_run is enabled' ' or in custom deployments (default: %(default)s)', dest='db_url', - default=constants.DEFAULT_DB_PROD_URL, type=str, metavar='PATH', ) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 3da432b1d..4e1137f33 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -110,9 +110,12 @@ class Configuration(object): '(not applicable with Backtesting and Hyperopt)' ) - if self.args.db_url != constants.DEFAULT_DB_PROD_URL: + if self.args.db_url and self.args.db_url != constants.DEFAULT_DB_PROD_URL: config.update({'db_url': self.args.db_url}) logger.info('Parameter --db-url detected ...') + else: + # Set default here + config.update({'db_url': constants.DEFAULT_DB_PROD_URL}) if config.get('dry_run', False): logger.info('Dry run is enabled') From 4d5e368c2ead3813877e761c66a54b400c8bb329 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Sep 2018 19:40:32 +0200 Subject: [PATCH 7/9] Remove direct call to pytest fixture to elliminate pytest warning --- freqtrade/tests/test_acl_pair.py | 47 ++++++++++++++++---------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 535684b22..38df3cb38 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -2,38 +2,39 @@ from unittest.mock import MagicMock -import freqtrade.tests.conftest as tt # test tools +from freqtrade.tests.conftest import get_patched_freqtradebot + +import pytest # whitelist, blacklist, filtering, all of that will # eventually become some rules to run on a generic ACL engine # perhaps try to anticipate that by using some python package -def whitelist_conf(): - config = tt.default_conf() - config['stake_currency'] = 'BTC' - config['exchange']['pair_whitelist'] = [ +@pytest.fixture(scope="function") +def whitelist_conf(default_conf): + default_conf['stake_currency'] = 'BTC' + default_conf['exchange']['pair_whitelist'] = [ 'ETH/BTC', 'TKN/BTC', 'TRST/BTC', 'SWT/BTC', 'BCC/BTC' ] - config['exchange']['pair_blacklist'] = [ + default_conf['exchange']['pair_blacklist'] = [ 'BLK/BTC' ] - return config + return default_conf -def test_refresh_market_pair_not_in_whitelist(mocker, markets): - conf = whitelist_conf() +def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): - freqtradebot = tt.get_patched_freqtradebot(mocker, conf) + freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) refreshedwhitelist = freqtradebot._refresh_whitelist( - conf['exchange']['pair_whitelist'] + ['XXX/BTC'] + whitelist_conf['exchange']['pair_whitelist'] + ['XXX/BTC'] ) # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] @@ -41,12 +42,12 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets): assert whitelist == refreshedwhitelist -def test_refresh_whitelist(mocker, markets): - conf = whitelist_conf() - freqtradebot = tt.get_patched_freqtradebot(mocker, conf) +def test_refresh_whitelist(mocker, markets, whitelist_conf): + freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - refreshedwhitelist = freqtradebot._refresh_whitelist(conf['exchange']['pair_whitelist']) + refreshedwhitelist = freqtradebot._refresh_whitelist( + whitelist_conf['exchange']['pair_whitelist']) # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] @@ -54,9 +55,8 @@ def test_refresh_whitelist(mocker, markets): assert whitelist == refreshedwhitelist -def test_refresh_whitelist_dynamic(mocker, markets, tickers): - conf = whitelist_conf() - freqtradebot = tt.get_patched_freqtradebot(mocker, conf) +def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): + freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_markets=markets, @@ -68,21 +68,20 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers): whitelist = ['ETH/BTC', 'TKN/BTC'] refreshedwhitelist = freqtradebot._refresh_whitelist( - freqtradebot._gen_pair_whitelist(conf['stake_currency']) + freqtradebot._gen_pair_whitelist(whitelist_conf['stake_currency']) ) assert whitelist == refreshedwhitelist -def test_refresh_whitelist_dynamic_empty(mocker, markets_empty): - conf = whitelist_conf() - freqtradebot = tt.get_patched_freqtradebot(mocker, conf) +def test_refresh_whitelist_dynamic_empty(mocker, markets_empty, whitelist_conf): + freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets_empty) # argument: use the whitelist dynamically by exchange-volume whitelist = [] - conf['exchange']['pair_whitelist'] = [] + whitelist_conf['exchange']['pair_whitelist'] = [] freqtradebot._refresh_whitelist(whitelist) - pairslist = conf['exchange']['pair_whitelist'] + pairslist = whitelist_conf['exchange']['pair_whitelist'] assert set(whitelist) == set(pairslist) From 88ccdc03660ffeac25e033a07bce27807cd07e0e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Sep 2018 20:45:01 +0200 Subject: [PATCH 8/9] Fix exception when order cannot be found --- freqtrade/freqtradebot.py | 4 +-- freqtrade/tests/test_freqtradebot.py | 38 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5c943ba3d..fa803bda7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -10,7 +10,7 @@ from datetime import datetime from typing import Any, Callable, Dict, List, Optional import arrow -import requests +from requests.exceptions import RequestException from cachetools import TTLCache, cached @@ -646,7 +646,7 @@ class FreqtradeBot(object): if not trade.open_order_id: continue order = self.exchange.get_order(trade.open_order_id, trade.pair) - except requests.exceptions.RequestException: + except (RequestException, DependencyException): logger.info( 'Cannot query order for %s due to %s', trade, diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 5e982f11a..1392d4091 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -993,6 +993,44 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe assert nb_trades == 0 +def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_order_old, + fee, mocker) -> None: + rpc_mock = patch_RPCManager(mocker) + cancel_order_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_order=MagicMock(side_effect=DependencyException), + cancel_order=cancel_order_mock, + get_fee=fee + ) + freqtrade = FreqtradeBot(default_conf) + + trade_buy = Trade( + pair='ETH/BTC', + open_rate=0.00001099, + exchange='bittrex', + open_order_id='123456789', + amount=90.99181073, + fee_open=0.0, + fee_close=0.0, + stake_amount=1, + open_date=arrow.utcnow().shift(minutes=-601).datetime, + is_open=True + ) + + Trade.session.add(trade_buy) + + # check it does cancel buy orders over the time limit + freqtrade.check_handle_timedout() + assert cancel_order_mock.call_count == 0 + assert rpc_mock.call_count == 0 + trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() + nb_trades = len(trades) + assert nb_trades == 1 + + def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() From f790f953191239f7b477576686a2ffea43ec91d6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 26 Sep 2018 14:28:07 +0200 Subject: [PATCH 9/9] Update ccxt from 1.17.350 to 1.17.351 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 74b3df91d..59d06aca1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.350 +ccxt==1.17.351 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1