From 0440a1917169eb9ed2e6fde713903fe9baec311a Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 14:19:50 +0200 Subject: [PATCH 01/25] export open/close rate for backtesting too preparation to allow plotting of backtest results --- freqtrade/optimize/backtesting.py | 22 +++++++++++++------- freqtrade/tests/optimize/test_backtesting.py | 18 +++++++++++----- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ffb808a24..b5ed9b167 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -37,6 +37,8 @@ class BacktestResult(NamedTuple): close_index: int trade_duration: float open_at_end: bool + open_rate: float + close_rate: float class Backtesting(object): @@ -115,11 +117,13 @@ class Backtesting(object): def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: - records = [(trade_entry.pair, trade_entry.profit_percent, - trade_entry.open_time.timestamp(), - trade_entry.close_time.timestamp(), - trade_entry.open_index - 1, trade_entry.trade_duration) - for index, trade_entry in results.iterrows()] + # columns = ["pair", "profit", "opents", "closets", "index", "duration", + # "open_rate", "close_rate", "open_at_end"] + + records = [(t.pair, t.profit_percent, t.open_time.timestamp(), + t.close_time.timestamp(), t.open_index - 1, t.trade_duration, + t.open_rate, t.close_rate, t.open_at_end) + for index, t in results.iterrows()] if records: logger.info('Dumping backtest results to %s', recordfilename) @@ -158,7 +162,9 @@ class Backtesting(object): trade_duration=(sell_row.date - buy_row.date).seconds // 60, open_index=buy_row.Index, close_index=sell_row.Index, - open_at_end=False + open_at_end=False, + open_rate=buy_row.close, + close_rate=sell_row.close ) if partial_ticker: # no sell condition found - trade stil open at end of backtest period @@ -171,7 +177,9 @@ class Backtesting(object): trade_duration=(sell_row.date - buy_row.date).seconds // 60, open_index=buy_row.Index, close_index=sell_row.Index, - open_at_end=True + open_at_end=True, + open_rate=buy_row.close, + close_rate=sell_row.close ) logger.debug('Force_selling still open trade %s with %s perc - %s', btr.pair, btr.profit_percent, btr.profit_abs) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 65aa00a70..4dda82463 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -604,9 +604,13 @@ def test_backtest_record(default_conf, fee, mocker): Arrow(2017, 11, 14, 22, 10, 00).datetime, Arrow(2017, 11, 14, 22, 43, 00).datetime, Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], + "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], "open_index": [1, 119, 153, 185], "close_index": [118, 151, 184, 199], - "trade_duration": [123, 34, 31, 14]}) + "trade_duration": [123, 34, 31, 14], + "open_at_end": [False, False, False, True] + }) backtesting._store_backtest_result("backtest-result.json", results) assert len(results) == 4 # Assert file_dump_json was only called once @@ -617,12 +621,16 @@ def test_backtest_record(default_conf, fee, mocker): # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117) # Below follows just a typecheck of the schema/type of trade-records oix = None - for (pair, profit, date_buy, date_sell, buy_index, dur) in records: + for (pair, profit, date_buy, date_sell, buy_index, dur, + openr, closer, open_at_end) in records: assert pair == 'UNITTEST/BTC' - isinstance(profit, float) + assert isinstance(profit, float) # FIX: buy/sell should be converted to ints - isinstance(date_buy, str) - isinstance(date_sell, str) + assert isinstance(date_buy, float) + assert isinstance(date_sell, float) + assert isinstance(openr, float) + assert isinstance(closer, float) + assert isinstance(open_at_end, bool) isinstance(buy_index, pd._libs.tslib.Timestamp) if oix: assert buy_index > oix From 3cedace2f6fce431cd5df784b5bcf5ce499bf02d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 19:54:27 +0200 Subject: [PATCH 02/25] add plotting for backtested trades --- scripts/plot_dataframe.py | 49 +++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 7f9641222..cd41f51b1 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -27,9 +27,12 @@ Example of usage: import logging import os import sys +import json +from pathlib import Path from argparse import Namespace from typing import Dict, List, Any +import pandas as pd import plotly.graph_objs as go from plotly import tools from plotly.offline import plot @@ -103,10 +106,42 @@ def plot_analyzed_dataframe(args: Namespace) -> None: exit() # Get trades already made from the DB - trades: List[Trade] = [] + trades: pd.DataFrame = pd.DataFrame() if args.db_url: persistence.init(_CONF) - trades = Trade.query.filter(Trade.pair.is_(pair)).all() + trades_ = Trade.query.filter(Trade.pair.is_(pair)).all() + # columns = ["pair", "profit", "opents", "closets", "index", "duration"] + columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] + + trades = pd.DataFrame([(t.pair, t.calc_profit(), + t.open_date, t.close_date, + t.open_rate, t.close_rate, + t.close_date.timestamp() - t.open_date.timestamp()) + for t in trades_], columns=columns) + + if args.exportfilename: + file = Path(args.exportfilename) + # must align with columns in backtest.py + columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end"] + with file.open() as f: + data = json.load(f) + trades = pd.DataFrame(data, columns=columns) + trades = trades.loc[trades["pair"] == pair] + if timerange: + if timerange.starttype == 'date': + trades = trades.loc[trades["opents"] >= timerange.startts] + if timerange.stoptype == 'date': + trades = trades.loc[trades["opents"] <= timerange.stopts] + + trades['opents'] = pd.to_datetime(trades['opents'], + unit='s', + utc=True, + infer_datetime_format=True) + trades['closets'] = pd.to_datetime(trades['closets'], + unit='s', + utc=True, + infer_datetime_format=True) dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] @@ -126,7 +161,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html')) -def generate_graph(pair, trades, data, args) -> tools.make_subplots: +def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots: """ Generate the graph from the data generated by Backtesting or from DB :param pair: Pair to Display on the graph @@ -187,8 +222,8 @@ def generate_graph(pair, trades, data, args) -> tools.make_subplots: ) trade_buys = go.Scattergl( - x=[t.open_date.isoformat() for t in trades], - y=[t.open_rate for t in trades], + x=trades["opents"], + y=trades["open_rate"], mode='markers', name='trade_buy', marker=dict( @@ -199,8 +234,8 @@ def generate_graph(pair, trades, data, args) -> tools.make_subplots: ) ) trade_sells = go.Scattergl( - x=[t.close_date.isoformat() for t in trades], - y=[t.close_rate for t in trades], + x=trades["closets"], + y=trades["close_rate"], mode='markers', name='trade_sell', marker=dict( From f506ebcd62bf26b522afdd658dddd1fa3371deac Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 19:58:28 +0200 Subject: [PATCH 03/25] use Pathlib in the whole script --- scripts/plot_dataframe.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index cd41f51b1..763b8d9e4 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -25,7 +25,6 @@ Example of usage: --indicators2 fastk,fastd """ import logging -import os import sys import json from pathlib import Path @@ -158,7 +157,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: args=args ) - plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html')) + plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html'))) def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots: From 5055563458597316bee1fbfafee1d42ba5eff673 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 20:14:15 +0200 Subject: [PATCH 04/25] add --plot-limit --- scripts/plot_dataframe.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 763b8d9e4..ba4da444a 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -104,19 +104,20 @@ def plot_analyzed_dataframe(args: Namespace) -> None: if tickers == {}: exit() + if args.db_url and args.exportfilename: + logger.critical("Can only specify --db-url or --export-filename") # Get trades already made from the DB trades: pd.DataFrame = pd.DataFrame() if args.db_url: persistence.init(_CONF) - trades_ = Trade.query.filter(Trade.pair.is_(pair)).all() - # columns = ["pair", "profit", "opents", "closets", "index", "duration"] columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] trades = pd.DataFrame([(t.pair, t.calc_profit(), t.open_date, t.close_date, t.open_rate, t.close_rate, t.close_date.timestamp() - t.open_date.timestamp()) - for t in trades_], columns=columns) + for t in Trade.query.filter(Trade.pair.is_(pair)).all()], + columns=columns) if args.exportfilename: file = Path(args.exportfilename) @@ -147,13 +148,15 @@ def plot_analyzed_dataframe(args: Namespace) -> None: dataframe = analyze.populate_buy_trend(dataframe) dataframe = analyze.populate_sell_trend(dataframe) - if len(dataframe.index) > 750: - logger.warning('Ticker contained more than 750 candles, clipping.') - + if len(dataframe.index) > args.plot_limit: + logger.warning('Ticker contained more than %s candles as defined ' + 'with --plot-limit, clipping.', args.plot_limit) + dataframe = dataframe.tail(args.plot_limit) + trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']] fig = generate_graph( pair=pair, trades=trades, - data=dataframe.tail(750), + data=dataframe, args=args ) @@ -333,11 +336,17 @@ def plot_parse_args(args: List[str]) -> Namespace: default='macd', dest='indicators2', ) - + arguments.parser.add_argument( + '--plot-limit', + help='Specify tick limit for plotting - too high values cause huge files - ' + 'Default: %(default)s', + dest='plot_limit', + default=750, + type=str, + ) arguments.common_args_parser() arguments.optimizer_shared_options(arguments.parser) arguments.backtesting_options(arguments.parser) - return arguments.parse_args() From d8cb63efdde8d118b4939dda1d4e7a796a6402a6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 20:19:07 +0200 Subject: [PATCH 05/25] extract load_trades --- scripts/plot_dataframe.py | 77 +++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index ba4da444a..e7e38fcd3 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -39,7 +39,7 @@ from plotly.offline import plot import freqtrade.optimize as optimize from freqtrade import persistence from freqtrade.analyze import Analyze -from freqtrade.arguments import Arguments +from freqtrade.arguments import Arguments, TimeRange from freqtrade.exchange import Exchange from freqtrade.optimize.backtesting import setup_configuration from freqtrade.persistence import Trade @@ -48,6 +48,45 @@ logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} +def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame: + trades: pd.DataFrame = pd.DataFrame() + if args.db_url: + persistence.init(_CONF) + columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] + + trades = pd.DataFrame([(t.pair, t.calc_profit(), + t.open_date, t.close_date, + t.open_rate, t.close_rate, + t.close_date.timestamp() - t.open_date.timestamp()) + for t in Trade.query.filter(Trade.pair.is_(pair)).all()], + columns=columns) + + if args.exportfilename: + file = Path(args.exportfilename) + # must align with columns in backtest.py + columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end"] + with file.open() as f: + data = json.load(f) + trades = pd.DataFrame(data, columns=columns) + trades = trades.loc[trades["pair"] == pair] + if timerange: + if timerange.starttype == 'date': + trades = trades.loc[trades["opents"] >= timerange.startts] + if timerange.stoptype == 'date': + trades = trades.loc[trades["opents"] <= timerange.stopts] + + trades['opents'] = pd.to_datetime(trades['opents'], + unit='s', + utc=True, + infer_datetime_format=True) + trades['closets'] = pd.to_datetime(trades['closets'], + unit='s', + utc=True, + infer_datetime_format=True) + return trades + + def plot_analyzed_dataframe(args: Namespace) -> None: """ Calls analyze() and plots the returned dataframe @@ -107,41 +146,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: if args.db_url and args.exportfilename: logger.critical("Can only specify --db-url or --export-filename") # Get trades already made from the DB - trades: pd.DataFrame = pd.DataFrame() - if args.db_url: - persistence.init(_CONF) - columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] - - trades = pd.DataFrame([(t.pair, t.calc_profit(), - t.open_date, t.close_date, - t.open_rate, t.close_rate, - t.close_date.timestamp() - t.open_date.timestamp()) - for t in Trade.query.filter(Trade.pair.is_(pair)).all()], - columns=columns) - - if args.exportfilename: - file = Path(args.exportfilename) - # must align with columns in backtest.py - columns = ["pair", "profit", "opents", "closets", "index", "duration", - "open_rate", "close_rate", "open_at_end"] - with file.open() as f: - data = json.load(f) - trades = pd.DataFrame(data, columns=columns) - trades = trades.loc[trades["pair"] == pair] - if timerange: - if timerange.starttype == 'date': - trades = trades.loc[trades["opents"] >= timerange.startts] - if timerange.stoptype == 'date': - trades = trades.loc[trades["opents"] <= timerange.stopts] - - trades['opents'] = pd.to_datetime(trades['opents'], - unit='s', - utc=True, - infer_datetime_format=True) - trades['closets'] = pd.to_datetime(trades['closets'], - unit='s', - utc=True, - infer_datetime_format=True) + trades = load_trades(args, pair, timerange) dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] From 660ec6f443046f6a2f02a2ec23ccd2a4ad4f9658 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 24 Jun 2018 13:43:27 +0200 Subject: [PATCH 06/25] fix parameter type --- scripts/plot_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index e7e38fcd3..1cc6b818a 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -347,7 +347,7 @@ def plot_parse_args(args: List[str]) -> Namespace: 'Default: %(default)s', dest='plot_limit', default=750, - type=str, + type=int, ) arguments.common_args_parser() arguments.optimizer_shared_options(arguments.parser) From e70cb963f7cc9f82336c2fa07875b97d31e74c94 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 24 Jun 2018 17:00:00 +0200 Subject: [PATCH 07/25] document what to do with exported backtest results --- docs/backtesting.md | 28 ++++++++++++++++++++++++++++ freqtrade/optimize/backtesting.py | 3 --- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 1efb46b43..172969ae2 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -70,6 +70,34 @@ Where `-s TestStrategy` refers to the class name within the strategy file `test_ python3 ./freqtrade/main.py backtesting --export trades ``` +The exported trades can be read using the following code for manual analysis, or can be used by the plotting script `plot_dataframe.py` in the scripts folder. + +``` python +import json +from pathlib import Path +import pandas as pd + +filename=Path('user_data/backtest_data/backtest-result.json') + +with filename.open() as file: + data = json.load(file) + +columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end"] +df = pd.DataFrame(data, columns=columns) + +df['opents'] = pd.to_datetime(df['opents'], + unit='s', + utc=True, + infer_datetime_format=True + ) +df['closets'] = pd.to_datetime(df['closets'], + unit='s', + utc=True, + infer_datetime_format=True + ) +``` + #### Exporting trades to file specifying a custom filename ```bash diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index b5ed9b167..70f6168a4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -117,9 +117,6 @@ class Backtesting(object): def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: - # columns = ["pair", "profit", "opents", "closets", "index", "duration", - # "open_rate", "close_rate", "open_at_end"] - records = [(t.pair, t.profit_percent, t.open_time.timestamp(), t.close_time.timestamp(), t.open_index - 1, t.trade_duration, t.open_rate, t.close_rate, t.open_at_end) From 19beb0941f4354e00a89238ff1630d86d5df9050 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 27 Jun 2018 14:23:07 +0200 Subject: [PATCH 08/25] Update ccxt from 1.14.272 to 1.14.288 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d4f8b2f06..62e2e3983 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.272 +ccxt==1.14.288 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 7cecae52799eb480e2265802e41ef0bd863d5a1c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 28 Jun 2018 14:23:07 +0200 Subject: [PATCH 09/25] Update ccxt from 1.14.288 to 1.14.289 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 62e2e3983..76dba3c49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.288 +ccxt==1.14.289 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 2d4ce593b511e3f9cd977a307c002307b4c6a052 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 19:48:05 +0200 Subject: [PATCH 10/25] catch crash with cobinhood fixes #959 --- freqtrade/freqtradebot.py | 11 +++++++---- freqtrade/tests/test_freqtradebot.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e25ed66cf..985ecb370 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -277,11 +277,14 @@ class FreqtradeBot(object): return None min_stake_amounts = [] - if 'cost' in market['limits'] and 'min' in market['limits']['cost']: - min_stake_amounts.append(market['limits']['cost']['min']) + limits = market['limits'] + if ('cost' in limits and 'min' in limits['cost'] + and limits['cost']['min'] is not None): + min_stake_amounts.append(limits['cost']['min']) - if 'amount' in market['limits'] and 'min' in market['limits']['amount']: - min_stake_amounts.append(market['limits']['amount']['min'] * price) + if ('amount' in limits and 'min' in limits['amount'] + and limits['amount']['min'] is not None): + min_stake_amounts.append(limits['amount']['min'] * price) if not min_stake_amounts: return None diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..ff1b7161d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -348,6 +348,34 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) assert result is None + # no cost Min + mocker.patch( + 'freqtrade.exchange.Exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {"min": None}, + 'amount': {} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) + assert result is None + + # no amount Min + mocker.patch( + 'freqtrade.exchange.Exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {}, + 'amount': {"min": None} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) + assert result is None + # empty 'cost'/'amount' section mocker.patch( 'freqtrade.exchange.Exchange.get_markets', From 8ec9a0974978242fb1bd47587151eec6a0a5bda6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 21:22:43 +0200 Subject: [PATCH 11/25] Standardize retrier exception testing --- freqtrade/tests/exchange/test_exchange.py | 141 ++++++---------------- 1 file changed, 39 insertions(+), 102 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 53e59b34b..4d46d7e61 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -14,6 +14,22 @@ from freqtrade.exchange import Exchange, API_RETRY_COUNT from freqtrade.tests.conftest import log_has, get_patched_exchange +def validate_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): + """Function to test exception handling """ + + with pytest.raises(TemporaryError): + api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[innerfun].call_count == API_RETRY_COUNT + 1 + + with pytest.raises(OperationalException): + api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[innerfun].call_count == 1 + + def test_init(default_conf, mocker, caplog): caplog.set_level(logging.INFO) get_patched_exchange(mocker, default_conf) @@ -243,17 +259,8 @@ def test_get_balances_prod(default_conf, mocker): assert exchange.get_balances()['1ST']['total'] == 10.0 assert exchange.get_balances()['1ST']['used'] == 0.0 - with pytest.raises(TemporaryError): - api_mock.fetch_balance = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_balances() - assert api_mock.fetch_balance.call_count == API_RETRY_COUNT + 1 - - with pytest.raises(OperationalException): - api_mock.fetch_balance = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_balances() - assert api_mock.fetch_balance.call_count == 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_balances", "fetch_balance") def test_get_tickers(default_conf, mocker): @@ -282,15 +289,8 @@ def test_get_tickers(default_conf, mocker): assert tickers['BCH/BTC']['bid'] == 0.6 assert tickers['BCH/BTC']['ask'] == 0.5 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_tickers() - - with pytest.raises(OperationalException): - api_mock.fetch_tickers = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_tickers() + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_tickers", "fetch_tickers") with pytest.raises(OperationalException): api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NotSupported) @@ -345,15 +345,9 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=False) assert api_mock.fetch_ticker.call_count == 0 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_ticker = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_ticker(pair='ETH/BTC', refresh=True) - - with pytest.raises(OperationalException): - api_mock.fetch_ticker = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_ticker(pair='ETH/BTC', refresh=True) + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker", "fetch_ticker", + pair='ETH/BTC', refresh=True) api_mock.fetch_ticker = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock) @@ -416,17 +410,9 @@ def test_get_ticker_history(default_conf, mocker): assert ticks[0][4] == 9 assert ticks[0][5] == 10 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - # new symbol to get around cache - exchange.get_ticker_history('ABCD/BTC', default_conf['ticker_interval']) - - with pytest.raises(OperationalException): - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - # new symbol to get around cache - exchange.get_ticker_history('EFGH/BTC', default_conf['ticker_interval']) + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker_history", "fetch_ohlcv", + pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) def test_get_ticker_history_sort(default_conf, mocker): @@ -515,24 +501,15 @@ def test_cancel_order(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) assert exchange.cancel_order(order_id='_', pair='TKN/BTC') == 123 - with pytest.raises(TemporaryError): - api_mock.cancel_order = MagicMock(side_effect=ccxt.NetworkError) - - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.cancel_order(order_id='_', pair='TKN/BTC') - assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(DependencyException): api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.cancel_order(order_id='_', pair='TKN/BTC') assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(OperationalException): - api_mock.cancel_order = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.cancel_order(order_id='_', pair='TKN/BTC') - assert api_mock.cancel_order.call_count == 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + "cancel_order", "cancel_order", + order_id='_', pair='TKN/BTC') def test_get_order(default_conf, mocker): @@ -550,23 +527,15 @@ def test_get_order(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) assert exchange.get_order('X', 'TKN/BTC') == 456 - with pytest.raises(TemporaryError): - api_mock.fetch_order = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_order(order_id='_', pair='TKN/BTC') - assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(DependencyException): api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_order(order_id='_', pair='TKN/BTC') assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(OperationalException): - api_mock.fetch_order = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_order(order_id='_', pair='TKN/BTC') - assert api_mock.fetch_order.call_count == 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_order', 'fetch_order', + order_id='_', pair='TKN/BTC') def test_name(default_conf, mocker): @@ -651,19 +620,9 @@ def test_get_trades_for_order(default_conf, mocker): assert len(orders) == 1 assert orders[0]['price'] == 165 - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_trades_for_order(order_id, 'LTC/BTC', since) - - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_trades_for_order(order_id, 'LTC/BTC', since) - assert api_mock.fetch_my_trades.call_count == API_RETRY_COUNT + 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_trades_for_order', 'fetch_my_trades', + order_id=order_id, pair='LTC/BTC', since=since) def test_get_markets(default_conf, mocker, markets): @@ -677,19 +636,8 @@ def test_get_markets(default_conf, mocker, markets): assert ret[0]["id"] == "ethbtc" assert ret[0]["symbol"] == "ETH/BTC" - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.fetch_markets = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_markets() - - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.fetch_markets = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_markets() - assert api_mock.fetch_markets.call_count == API_RETRY_COUNT + 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_markets', 'fetch_markets') def test_get_fee(default_conf, mocker): @@ -704,19 +652,8 @@ def test_get_fee(default_conf, mocker): assert exchange.get_fee() == 0.025 - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.calculate_fee = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_fee() - - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.calculate_fee = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_fee() - assert api_mock.calculate_fee.call_count == API_RETRY_COUNT + 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_fee', 'calculate_fee') def test_get_amount_lots(default_conf, mocker): From ebbfc720b24c6a378dcf79b5b1c5205f8dd95dea Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 21:51:59 +0200 Subject: [PATCH 12/25] increase test coverage --- freqtrade/tests/exchange/test_exchange.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 4d46d7e61..75394ca84 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -232,6 +232,12 @@ def test_get_balance_prod(default_conf, mocker): exchange.get_balance(currency='BTC') + with pytest.raises(TemporaryError): + # api_mock.fetch_balance = MagicMock(return_value={}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + mocker.patch('freqtrade.exchange.Exchange.get_balances', MagicMock(return_value={})) + exchange.get_balance(currency='BTC') + def test_get_balances_dry_run(default_conf, mocker): default_conf['dry_run'] = True @@ -624,6 +630,9 @@ def test_get_trades_for_order(default_conf, mocker): 'get_trades_for_order', 'fetch_my_trades', order_id=order_id, pair='LTC/BTC', since=since) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) + assert exchange.get_trades_for_order(order_id, 'LTC/BTC', since) == [] + def test_get_markets(default_conf, mocker, markets): api_mock = MagicMock() From fe8a21681e3fd8af90093a9fd004e7acd2c51f77 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 21:56:37 +0200 Subject: [PATCH 13/25] add test for Not supported --- freqtrade/tests/exchange/test_exchange.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 75394ca84..012c55356 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -232,8 +232,7 @@ def test_get_balance_prod(default_conf, mocker): exchange.get_balance(currency='BTC') - with pytest.raises(TemporaryError): - # api_mock.fetch_balance = MagicMock(return_value={}) + with pytest.raises(TemporaryError, match=r'.*balance due to malformed exchange response:.*'): exchange = get_patched_exchange(mocker, default_conf, api_mock) mocker.patch('freqtrade.exchange.Exchange.get_balances', MagicMock(return_value={})) exchange.get_balance(currency='BTC') @@ -420,6 +419,11 @@ def test_get_ticker_history(default_conf, mocker): "get_ticker_history", "fetch_ohlcv", pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): + api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_ticker_history(pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + def test_get_ticker_history_sort(default_conf, mocker): api_mock = MagicMock() From 15c7854e7f79c03132e86d789f7725108aec4529 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 22:11:45 +0200 Subject: [PATCH 14/25] add test for exchange_has --- freqtrade/tests/exchange/test_exchange.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 012c55356..25c6fa0c2 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -113,6 +113,20 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog): Exchange(conf) +def test_exchangehas(default_conf, mocker): + exchange = get_patched_exchange(mocker, default_conf) + assert not exchange.exchange_has('ASDFASDF') + api_mock = MagicMock() + + type(api_mock).has = PropertyMock(return_value={'deadbeef': True}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + assert exchange.exchange_has("deadbeef") + + type(api_mock).has = PropertyMock(return_value={'deadbeef': False}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + assert not exchange.exchange_has("deadbeef") + + def test_buy_dry_run(default_conf, mocker): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf) From dcdc18a33868a5630b5b747eb2773819154fb88c Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 22:18:38 +0200 Subject: [PATCH 15/25] rename test-function --- freqtrade/tests/exchange/test_exchange.py | 50 +++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 25c6fa0c2..712205893 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -14,8 +14,8 @@ from freqtrade.exchange import Exchange, API_RETRY_COUNT from freqtrade.tests.conftest import log_has, get_patched_exchange -def validate_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): - """Function to test exception handling """ +def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): + """Function to test ccxt exception handling """ with pytest.raises(TemporaryError): api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.NetworkError) @@ -278,8 +278,8 @@ def test_get_balances_prod(default_conf, mocker): assert exchange.get_balances()['1ST']['total'] == 10.0 assert exchange.get_balances()['1ST']['used'] == 0.0 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_balances", "fetch_balance") + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_balances", "fetch_balance") def test_get_tickers(default_conf, mocker): @@ -308,8 +308,8 @@ def test_get_tickers(default_conf, mocker): assert tickers['BCH/BTC']['bid'] == 0.6 assert tickers['BCH/BTC']['ask'] == 0.5 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_tickers", "fetch_tickers") + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_tickers", "fetch_tickers") with pytest.raises(OperationalException): api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NotSupported) @@ -364,9 +364,9 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=False) assert api_mock.fetch_ticker.call_count == 0 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_ticker", "fetch_ticker", - pair='ETH/BTC', refresh=True) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker", "fetch_ticker", + pair='ETH/BTC', refresh=True) api_mock.fetch_ticker = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock) @@ -429,9 +429,9 @@ def test_get_ticker_history(default_conf, mocker): assert ticks[0][4] == 9 assert ticks[0][5] == 10 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_ticker_history", "fetch_ohlcv", - pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker_history", "fetch_ohlcv", + pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) @@ -531,9 +531,9 @@ def test_cancel_order(default_conf, mocker): exchange.cancel_order(order_id='_', pair='TKN/BTC') assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "cancel_order", "cancel_order", - order_id='_', pair='TKN/BTC') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "cancel_order", "cancel_order", + order_id='_', pair='TKN/BTC') def test_get_order(default_conf, mocker): @@ -557,9 +557,9 @@ def test_get_order(default_conf, mocker): exchange.get_order(order_id='_', pair='TKN/BTC') assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_order', 'fetch_order', - order_id='_', pair='TKN/BTC') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_order', 'fetch_order', + order_id='_', pair='TKN/BTC') def test_name(default_conf, mocker): @@ -644,9 +644,9 @@ def test_get_trades_for_order(default_conf, mocker): assert len(orders) == 1 assert orders[0]['price'] == 165 - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_trades_for_order', 'fetch_my_trades', - order_id=order_id, pair='LTC/BTC', since=since) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_trades_for_order', 'fetch_my_trades', + order_id=order_id, pair='LTC/BTC', since=since) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) assert exchange.get_trades_for_order(order_id, 'LTC/BTC', since) == [] @@ -663,8 +663,8 @@ def test_get_markets(default_conf, mocker, markets): assert ret[0]["id"] == "ethbtc" assert ret[0]["symbol"] == "ETH/BTC" - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_markets', 'fetch_markets') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_markets', 'fetch_markets') def test_get_fee(default_conf, mocker): @@ -679,8 +679,8 @@ def test_get_fee(default_conf, mocker): assert exchange.get_fee() == 0.025 - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_fee', 'calculate_fee') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_fee', 'calculate_fee') def test_get_amount_lots(default_conf, mocker): From cf6b1a637ac0a2312c15d926093851a3a1adf4f1 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 22:32:28 +0200 Subject: [PATCH 16/25] increase exchange code coverage --- freqtrade/tests/exchange/test_exchange.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 712205893..99b960885 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -36,7 +36,7 @@ def test_init(default_conf, mocker, caplog): assert log_has('Instance is running with dry_run enabled', caplog.record_tuples) -def test_init_exception(default_conf): +def test_init_exception(default_conf, mocker): default_conf['exchange']['name'] = 'wrong_exchange_name' with pytest.raises( @@ -44,6 +44,13 @@ def test_init_exception(default_conf): match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): Exchange(default_conf) + default_conf['exchange']['name'] = 'binance' + with pytest.raises( + OperationalException, + match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): + mocker.patch("ccxt.binance", MagicMock(side_effect=AttributeError)) + Exchange(default_conf) + def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() From 8a941f3aa81ee6aad4778be0e1ae0952ce3108c9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 29 Jun 2018 14:23:06 +0200 Subject: [PATCH 17/25] Update ccxt from 1.14.289 to 1.14.295 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 76dba3c49..6995c2513 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.289 +ccxt==1.14.295 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 98108a78f1ff5ccd0ca4390a20885137f2d65637 Mon Sep 17 00:00:00 2001 From: Nullart Date: Thu, 14 Jun 2018 09:32:52 +0800 Subject: [PATCH 18/25] separating unfulfilled timeouts for buy and sell --- config.json.example | 5 ++++- config_full.json.example | 5 ++++- freqtrade/constants.py | 9 ++++++++- freqtrade/freqtradebot.py | 19 ++++++++++++------- freqtrade/tests/conftest.py | 5 ++++- freqtrade/tests/test_freqtradebot.py | 8 ++++---- 6 files changed, 36 insertions(+), 15 deletions(-) diff --git a/config.json.example b/config.json.example index e5bdc95b1..336a0c374 100644 --- a/config.json.example +++ b/config.json.example @@ -5,7 +5,10 @@ "fiat_display_currency": "USD", "ticker_interval" : "5m", "dry_run": false, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy":10, + "sell":30 + } "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/config_full.json.example b/config_full.json.example index 231384acc..1985edc70 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -12,7 +12,10 @@ "0": 0.04 }, "stoploss": -0.10, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy":10, + "sell":30 + } "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 0f12905e3..44bd64fe9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -61,7 +61,14 @@ CONF_SCHEMA = { 'minProperties': 1 }, 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, - 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, + 'unfilledtimeout': { + 'type': 'object', + 'properties': { + 'use_book_order': {'type': 'boolean'}, + 'buy': {'type': 'number', 'minimum': 3}, + 'sell': {'type': 'number', 'minimum': 10} + } + }, 'bid_strategy': { 'type': 'object', 'properties': { diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e25ed66cf..c5f932119 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -160,7 +160,7 @@ class FreqtradeBot(object): if 'unfilledtimeout' in self.config: # Check and handle any timed out open orders - self.check_handle_timedout(self.config['unfilledtimeout']) + self.check_handle_timedout() Trade.session.flush() except TemporaryError as error: @@ -492,13 +492,16 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False - def check_handle_timedout(self, timeoutvalue: int) -> None: + def check_handle_timedout(self) -> None: """ Check if any orders are timed out and cancel if neccessary :param timeoutvalue: Number of minutes until order is considered timed out :return: None """ - timeoutthreashold = arrow.utcnow().shift(minutes=-timeoutvalue).datetime + buy_timeout = self.config['unfilledtimeout']['buy'] + sell_timeout = self.config['unfilledtimeout']['sell'] + buy_timeoutthreashold = arrow.utcnow().shift(minutes=-buy_timeout).datetime + sell_timeoutthreashold = arrow.utcnow().shift(minutes=-sell_timeout).datetime for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all(): try: @@ -521,10 +524,12 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ if int(order['remaining']) == 0: continue - if order['side'] == 'buy' and ordertime < timeoutthreashold: - self.handle_timedout_limit_buy(trade, order) - elif order['side'] == 'sell' and ordertime < timeoutthreashold: - self.handle_timedout_limit_sell(trade, order) + # Check if trade is still actually open + if (order['status'] == 'open'): + if order['side'] == 'buy' and ordertime < buy_timeoutthreashold: + self.handle_timedout_limit_buy(trade, order) + elif order['side'] == 'sell' and ordertime < sell_timeoutthreashold: + self.handle_timedout_limit_sell(trade, order) # FIX: 20180110, why is cancel.order unconditionally here, whereas # it is conditionally called in the diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index d9ccaa325..4877fe556 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -100,7 +100,10 @@ def default_conf(): "0": 0.04 }, "stoploss": -0.10, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy": 10, + "sell": 30 + }, "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..fb7579d45 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1124,7 +1124,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe Trade.session.add(trade_buy) # check it does cancel buy orders over the time limit - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() @@ -1165,7 +1165,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, Trade.session.add(trade_sell) # check it does cancel sell orders over the time limit - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 assert trade_sell.is_open is True @@ -1205,7 +1205,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old # check it does cancel buy orders over the time limit # note this is for a partially-complete buy order - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() @@ -1256,7 +1256,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) - 'recent call last):\n.*' ) - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert filter(regexp.match, caplog.record_tuples) From c447644fd16386868c6a4a1aa6b8a5e139af8c22 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 30 Jun 2018 14:23:06 +0200 Subject: [PATCH 19/25] Update ccxt from 1.14.295 to 1.14.301 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6995c2513..72decf95a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.295 +ccxt==1.14.301 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 5a591e01c08fa0c60db152da2c04ec04dfd4e58a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 30 Jun 2018 14:23:07 +0200 Subject: [PATCH 20/25] Update sqlalchemy from 1.2.8 to 1.2.9 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 72decf95a..50441879a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ccxt==1.14.301 -SQLAlchemy==1.2.8 +SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 cachetools==2.1.0 From 14e12bd3c09c42a4ed20c170765a55826e304b04 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 30 Jun 2018 17:37:34 +0200 Subject: [PATCH 21/25] Fix missing comma in example.json --- config.json.example | 2 +- config_full.json.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json.example b/config.json.example index 336a0c374..606167ea3 100644 --- a/config.json.example +++ b/config.json.example @@ -8,7 +8,7 @@ "unfilledtimeout": { "buy":10, "sell":30 - } + }, "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/config_full.json.example b/config_full.json.example index 1985edc70..4bb794937 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -15,7 +15,7 @@ "unfilledtimeout": { "buy":10, "sell":30 - } + }, "bid_strategy": { "ask_last_balance": 0.0 }, From 9e3e900f78d5105f6d145d80fd5d97dbea0b9cac Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 30 Jun 2018 17:49:46 +0200 Subject: [PATCH 22/25] Add get_markets mock to new tests --- freqtrade/tests/test_freqtradebot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..f66864946 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1599,6 +1599,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) @@ -1616,7 +1617,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke assert freqtrade.handle_trade(trade) is True -def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> None: +def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: """ Test sell_profit_only feature when enabled and we have a loss """ @@ -1634,6 +1635,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) From 8f49d5eb1002230fb424e4b505ec5f9561e778ea Mon Sep 17 00:00:00 2001 From: Nullart Date: Thu, 14 Jun 2018 18:49:43 +0800 Subject: [PATCH 23/25] documentation updates --- docs/configuration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5279ed06e..ea3e99f15 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -22,7 +22,8 @@ The table below will list all configuration parameters. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. -| `unfilledtimeout` | 0 | No | How long (in minutes) the bot will wait for an unfilled order to complete, after which the order will be cancelled. +| `unfilledtimeout.buy` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. +| `unfilledtimeout.sell` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. From c66f858b988bd18ee24045839211f4248640b097 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 19:37:55 +0200 Subject: [PATCH 24/25] rename innerfun to mock_ccxt_fun --- freqtrade/tests/exchange/test_exchange.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 99b960885..6cf7f11b0 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -14,20 +14,20 @@ from freqtrade.exchange import Exchange, API_RETRY_COUNT from freqtrade.tests.conftest import log_has, get_patched_exchange -def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): +def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): """Function to test ccxt exception handling """ with pytest.raises(TemporaryError): - api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.NetworkError) + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) getattr(exchange, fun)(**kwargs) - assert api_mock.__dict__[innerfun].call_count == API_RETRY_COUNT + 1 + assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1 with pytest.raises(OperationalException): - api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.BaseError) + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) getattr(exchange, fun)(**kwargs) - assert api_mock.__dict__[innerfun].call_count == 1 + assert api_mock.__dict__[mock_ccxt_fun].call_count == 1 def test_init(default_conf, mocker, caplog): From 2dc881558d65b19dd6596d3a8847da618f7d6885 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 19:41:19 +0200 Subject: [PATCH 25/25] address PR comments --- config.json.example | 4 ++-- config_full.json.example | 4 ++-- freqtrade/constants.py | 1 - freqtrade/freqtradebot.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/config.json.example b/config.json.example index 606167ea3..79ba2ac64 100644 --- a/config.json.example +++ b/config.json.example @@ -6,8 +6,8 @@ "ticker_interval" : "5m", "dry_run": false, "unfilledtimeout": { - "buy":10, - "sell":30 + "buy": 10, + "sell": 30 }, "bid_strategy": { "ask_last_balance": 0.0 diff --git a/config_full.json.example b/config_full.json.example index 4bb794937..d40c11a91 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -13,8 +13,8 @@ }, "stoploss": -0.10, "unfilledtimeout": { - "buy":10, - "sell":30 + "buy": 10, + "sell": 30 }, "bid_strategy": { "ask_last_balance": 0.0 diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 44bd64fe9..04d4a44ac 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -64,7 +64,6 @@ CONF_SCHEMA = { 'unfilledtimeout': { 'type': 'object', 'properties': { - 'use_book_order': {'type': 'boolean'}, 'buy': {'type': 'number', 'minimum': 3}, 'sell': {'type': 'number', 'minimum': 10} } diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c5f932119..4d6352e55 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -525,7 +525,7 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ continue # Check if trade is still actually open - if (order['status'] == 'open'): + if order['status'] == 'open': if order['side'] == 'buy' and ordertime < buy_timeoutthreashold: self.handle_timedout_limit_buy(trade, order) elif order['side'] == 'sell' and ordertime < sell_timeoutthreashold: