From e8803477dfa269f9f95932f169c95554e22b504f Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Tue, 3 May 2022 22:56:55 -0600 Subject: [PATCH 01/30] exchange/exchange add param taker_or_maker to add_dry_order_fee --- freqtrade/exchange/exchange.py | 44 ++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 08bdab265..f0ff7e514 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -785,6 +785,26 @@ class Exchange: # Dry-run methods + def taker_or_maker( + self, + order_reason: Literal['entry', 'exit', 'stoploss'], # TODO: stoploss + ): + order_type = self._config['order_types'][order_reason] + if order_type == 'market' or order_reason == 'stoploss': + return 'taker' + else: + return ( + 'maker' if ( + ( + order_reason == 'entry' and + self._config['entry_pricing']['price_side'] == 'same' + ) or ( + order_reason == 'exit' and + self._config['exit_pricing']['price_side'] == 'same' + ) + ) else 'taker' + ) + def create_dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, leverage: float, params: Dict = {}, stop_loss: bool = False) -> Dict[str, Any]: @@ -824,7 +844,7 @@ class Exchange: 'filled': _amount, 'cost': (dry_order['amount'] * average) / leverage }) - dry_order = self.add_dry_order_fee(pair, dry_order) + dry_order = self.add_dry_order_fee(pair, dry_order, self.taker_or_maker('entry')) dry_order = self.check_dry_limit_order_filled(dry_order) @@ -832,12 +852,17 @@ class Exchange: # Copy order and close it - so the returned order is open unless it's a market order return dry_order - def add_dry_order_fee(self, pair: str, dry_order: Dict[str, Any]) -> Dict[str, Any]: + def add_dry_order_fee( + self, + pair: str, + dry_order: Dict[str, Any], + taker_or_maker: Literal['taker', 'maker'], + ) -> Dict[str, Any]: dry_order.update({ 'fee': { 'currency': self.get_pair_quote_currency(pair), - 'cost': dry_order['cost'] * self.get_fee(pair), - 'rate': self.get_fee(pair) + 'cost': dry_order['cost'] * self.get_fee(pair, taker_or_maker=taker_or_maker), + 'rate': self.get_fee(pair, taker_or_maker=taker_or_maker) } }) return dry_order @@ -917,7 +942,16 @@ class Exchange: 'filled': order['amount'], 'remaining': 0, }) - self.add_dry_order_fee(pair, order) + enter_long = not order['is_short'] and order['side'] == 'buy' + enter_short = order['is_short'] and order['side'] == 'sell' + entry_or_exit: Literal['entry', 'exit'] = ( + 'entry' if (enter_short or enter_long) else 'exit' + ) + self.add_dry_order_fee( + pair, + order, + self.taker_or_maker(entry_or_exit) + ) return order From 5d9aee6b7e6591ddb1953159223af8ebbb9cf6a1 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Tue, 3 May 2022 23:18:13 -0600 Subject: [PATCH 02/30] test_taker_or_maker --- tests/exchange/test_exchange.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 1368bcb85..3edb5187c 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -4957,3 +4957,24 @@ def test_stoploss_contract_size(mocker, default_conf, contract_size, order_amoun assert order['cost'] == 100 assert order['filled'] == 100 assert order['remaining'] == 100 + + +@pytest.mark.parametrize('order_reason,price_side,order_type,taker_or_maker', [ + ("entry", "same", "limit", "maker"), + ("exit", "same", "limit", "maker"), + ("entry", "other", "limit", "taker"), + ("exit", "other", "limit", "taker"), + ("entry", "same", "market", "maker"), + ("exit", "same", "market", "maker"), + ("entry", "other", "market", "taker"), + ("exit", "other", "market", "taker"), + ("stoploss", "same", "limit", "taker"), + ("stoploss", "same", "market", "taker"), + ("stoploss", "other", "limit", "taker"), + ("stoploss", "other", "market", "taker"), +]) +def test_taker_or_maker(mocker, default_conf, order_reason, price_side, order_type, taker_or_maker): + default_conf[f"{order_reason}_pricing"]["price_side"] = price_side + default_conf["order_types"][order_reason] = order_type + exchange = get_patched_exchange(mocker, default_conf) + assert exchange.taker_or_maker(order_reason) == taker_or_maker From dac9931b4a53aedeb4435483dcbb956203bb5530 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Tue, 3 May 2022 23:52:01 -0600 Subject: [PATCH 03/30] test_create_dry_run_order_fees --- tests/exchange/test_exchange.py | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 3edb5187c..ab51362a5 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1125,6 +1125,47 @@ def test_create_dry_run_order(default_conf, mocker, side, exchange_name, leverag assert order["cost"] == 1 * 200 / leverage +@pytest.mark.parametrize('side,is_short,order_reason', [ + ("buy", False, "entry"), + ("sell", False, "exit"), + ("buy", True, "exit"), + ("sell", True, "entry"), +]) +@pytest.mark.parametrize("order_type,price_side,fee", [ + ("limit", "same", 1.0), + ("limit", "other", 2.0), + ("market", "same", 2.0), + ("market", "other", 2.0), +]) +def test_create_dry_run_order_fees( + default_conf, + mocker, + side, + order_type, + is_short, + order_reason, + price_side, + fee, +): + default_conf[f"{order_reason}_pricing"]["price_side"] = "same" + default_conf["order_types"][order_reason] = order_type + mocker.patch( + 'freqtrade.exchange.Exchange.get_fee', + lambda symbol, taker_or_maker: 2.0 if taker_or_maker == 'taker' else 1.0 + ) + exchange = get_patched_exchange(mocker, default_conf) + + order = exchange.create_dry_run_order( + pair='ADA/USDT', + ordertype=order_type, + side=side, + amount=10, + rate=2.0, + ) + + assert order['ft_fee_base'] == fee + + @pytest.mark.parametrize("side,startprice,endprice", [ ("buy", 25.563, 25.566), ("sell", 25.566, 25.563) From 86ad5dd02a7ae680a1d566f7b83b55e5cf65f399 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 4 May 2022 00:08:41 -0600 Subject: [PATCH 04/30] test_exchange::test_taker_or_maker fixes --- tests/exchange/test_exchange.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index ab51362a5..a13dca4b6 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -5015,7 +5015,9 @@ def test_stoploss_contract_size(mocker, default_conf, contract_size, order_amoun ("stoploss", "other", "market", "taker"), ]) def test_taker_or_maker(mocker, default_conf, order_reason, price_side, order_type, taker_or_maker): - default_conf[f"{order_reason}_pricing"]["price_side"] = price_side + if order_reason != 'stoploss': + default_conf[f"{order_reason}_pricing"]["price_side"] = price_side + default_conf["order_types"] = {} default_conf["order_types"][order_reason] = order_type exchange = get_patched_exchange(mocker, default_conf) assert exchange.taker_or_maker(order_reason) == taker_or_maker From 10cbb5e67c32b184445f868366e4eef38afe5878 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 4 May 2022 00:10:09 -0600 Subject: [PATCH 05/30] test_exchange::test_taker_or_maker fixes --- tests/exchange/test_exchange.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index a13dca4b6..86928a2f6 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -5005,13 +5005,13 @@ def test_stoploss_contract_size(mocker, default_conf, contract_size, order_amoun ("exit", "same", "limit", "maker"), ("entry", "other", "limit", "taker"), ("exit", "other", "limit", "taker"), - ("entry", "same", "market", "maker"), - ("exit", "same", "market", "maker"), + ("stoploss", "same", "limit", "taker"), + ("stoploss", "other", "limit", "taker"), + ("entry", "same", "market", "taker"), + ("exit", "same", "market", "taker"), ("entry", "other", "market", "taker"), ("exit", "other", "market", "taker"), - ("stoploss", "same", "limit", "taker"), ("stoploss", "same", "market", "taker"), - ("stoploss", "other", "limit", "taker"), ("stoploss", "other", "market", "taker"), ]) def test_taker_or_maker(mocker, default_conf, order_reason, price_side, order_type, taker_or_maker): From 8b2535a8da3eabbbb53a70eeebb6348afaf0edd6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Jul 2022 15:42:17 +0200 Subject: [PATCH 06/30] Update Typing for fees --- freqtrade/constants.py | 1 + freqtrade/exchange/exchange.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index ce7c0ff83..6d74ceafd 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -542,3 +542,4 @@ TradeList = List[List] LongShort = Literal['long', 'short'] EntryExit = Literal['entry', 'exit'] BuySell = Literal['buy', 'sell'] +MakerTaker = Literal['maker', 'taker'] diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index db7a8ce41..2deb2b70d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -20,7 +20,7 @@ from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, Precise, decimal_to_ from pandas import DataFrame from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell, - EntryExit, ListPairsWithTimeframes, PairWithTimeframe) + EntryExit, ListPairsWithTimeframes, MakerTaker, PairWithTimeframe) from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, TradingMode from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError, @@ -870,7 +870,8 @@ class Exchange: 'filled': _amount, 'cost': (dry_order['amount'] * average) / leverage }) - dry_order = self.add_dry_order_fee(pair, dry_order, self.taker_or_maker('entry')) + # market orders will always incurr taker fees + dry_order = self.add_dry_order_fee(pair, dry_order, 'taker') dry_order = self.check_dry_limit_order_filled(dry_order) @@ -882,7 +883,7 @@ class Exchange: self, pair: str, dry_order: Dict[str, Any], - taker_or_maker: Literal['taker', 'maker'], + taker_or_maker: MakerTaker, ) -> Dict[str, Any]: dry_order.update({ 'fee': { @@ -970,7 +971,7 @@ class Exchange: }) enter_long = not order['is_short'] and order['side'] == 'buy' enter_short = order['is_short'] and order['side'] == 'sell' - entry_or_exit: Literal['entry', 'exit'] = ( + entry_or_exit: EntryExit = ( 'entry' if (enter_short or enter_long) else 'exit' ) self.add_dry_order_fee( @@ -1635,7 +1636,7 @@ class Exchange: @retrier def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, - price: float = 1, taker_or_maker: str = 'maker') -> float: + price: float = 1, taker_or_maker: MakerTaker = 'maker') -> float: try: if self._config['dry_run'] and self._config.get('fee', None) is not None: return self._config['fee'] From 4172f92bfc492a140103d79027fc326348183cc0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Jul 2022 17:03:45 +0200 Subject: [PATCH 07/30] simplify dry-run taker/maker selection --- freqtrade/exchange/exchange.py | 33 ++++--------------------- tests/exchange/test_exchange.py | 43 ++++++++++++--------------------- 2 files changed, 20 insertions(+), 56 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 2deb2b70d..efb33ee74 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -811,26 +811,6 @@ class Exchange: # Dry-run methods - def taker_or_maker( - self, - order_reason: Literal['entry', 'exit', 'stoploss'], # TODO: stoploss - ): - order_type = self._config['order_types'][order_reason] - if order_type == 'market' or order_reason == 'stoploss': - return 'taker' - else: - return ( - 'maker' if ( - ( - order_reason == 'entry' and - self._config['entry_pricing']['price_side'] == 'same' - ) or ( - order_reason == 'exit' and - self._config['exit_pricing']['price_side'] == 'same' - ) - ) else 'taker' - ) - def create_dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, leverage: float, params: Dict = {}, stop_loss: bool = False) -> Dict[str, Any]: @@ -873,7 +853,7 @@ class Exchange: # market orders will always incurr taker fees dry_order = self.add_dry_order_fee(pair, dry_order, 'taker') - dry_order = self.check_dry_limit_order_filled(dry_order) + dry_order = self.check_dry_limit_order_filled(dry_order, immediate=True) self._dry_run_open_orders[dry_order["id"]] = dry_order # Copy order and close it - so the returned order is open unless it's a market order @@ -955,7 +935,8 @@ class Exchange: pass return False - def check_dry_limit_order_filled(self, order: Dict[str, Any]) -> Dict[str, Any]: + def check_dry_limit_order_filled( + self, order: Dict[str, Any], immediate: bool = False) -> Dict[str, Any]: """ Check dry-run limit order fill and update fee (if it filled). """ @@ -969,15 +950,11 @@ class Exchange: 'filled': order['amount'], 'remaining': 0, }) - enter_long = not order['is_short'] and order['side'] == 'buy' - enter_short = order['is_short'] and order['side'] == 'sell' - entry_or_exit: EntryExit = ( - 'entry' if (enter_short or enter_long) else 'exit' - ) + self.add_dry_order_fee( pair, order, - self.taker_or_maker(entry_or_exit) + 'taker' if immediate else 'maker', ) return order diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 0c4df8f5a..ff8b4b40c 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1160,23 +1160,33 @@ def test_create_dry_run_order_fees( price_side, fee, ): - default_conf[f"{order_reason}_pricing"]["price_side"] = "same" - default_conf["order_types"][order_reason] = order_type mocker.patch( 'freqtrade.exchange.Exchange.get_fee', - lambda symbol, taker_or_maker: 2.0 if taker_or_maker == 'taker' else 1.0 + side_effect=lambda symbol, taker_or_maker: 2.0 if taker_or_maker == 'taker' else 1.0 ) + mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', + return_value=price_side == 'other') exchange = get_patched_exchange(mocker, default_conf) order = exchange.create_dry_run_order( - pair='ADA/USDT', + pair='LTC/USDT', ordertype=order_type, side=side, amount=10, rate=2.0, + leverage=1.0 ) + if price_side == 'other' or order_type == 'market': + assert order['fee']['rate'] == fee + return + else: + assert order['fee'] is None - assert order['ft_fee_base'] == fee + mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', + return_value=price_side != 'other') + + order1 = exchange.fetch_dry_run_order(order['id']) + assert order1['fee']['rate'] == fee @pytest.mark.parametrize("side,startprice,endprice", [ @@ -5101,26 +5111,3 @@ def test_stoploss_contract_size(mocker, default_conf, contract_size, order_amoun assert order['cost'] == 100 assert order['filled'] == 100 assert order['remaining'] == 100 - - -@pytest.mark.parametrize('order_reason,price_side,order_type,taker_or_maker', [ - ("entry", "same", "limit", "maker"), - ("exit", "same", "limit", "maker"), - ("entry", "other", "limit", "taker"), - ("exit", "other", "limit", "taker"), - ("stoploss", "same", "limit", "taker"), - ("stoploss", "other", "limit", "taker"), - ("entry", "same", "market", "taker"), - ("exit", "same", "market", "taker"), - ("entry", "other", "market", "taker"), - ("exit", "other", "market", "taker"), - ("stoploss", "same", "market", "taker"), - ("stoploss", "other", "market", "taker"), -]) -def test_taker_or_maker(mocker, default_conf, order_reason, price_side, order_type, taker_or_maker): - if order_reason != 'stoploss': - default_conf[f"{order_reason}_pricing"]["price_side"] = price_side - default_conf["order_types"] = {} - default_conf["order_types"][order_reason] = order_type - exchange = get_patched_exchange(mocker, default_conf) - assert exchange.taker_or_maker(order_reason) == taker_or_maker From 423af371c0ec8822f03cd9f10528e2c0d58e3512 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Jul 2022 17:59:05 +0200 Subject: [PATCH 08/30] Simplify calculation by calling "get_fee" only once --- freqtrade/exchange/exchange.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index efb33ee74..a430cdac5 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -865,11 +865,12 @@ class Exchange: dry_order: Dict[str, Any], taker_or_maker: MakerTaker, ) -> Dict[str, Any]: + fee = self.get_fee(pair, taker_or_maker=taker_or_maker) dry_order.update({ 'fee': { 'currency': self.get_pair_quote_currency(pair), - 'cost': dry_order['cost'] * self.get_fee(pair, taker_or_maker=taker_or_maker), - 'rate': self.get_fee(pair, taker_or_maker=taker_or_maker) + 'cost': dry_order['cost'] * fee, + 'rate': fee } }) return dry_order From 9347677c6061648517767dce5916dcf0394c71e6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Jul 2022 19:33:26 +0200 Subject: [PATCH 09/30] Uppdate pricecontours test to not recreate backtesting every loop in hopes to fix random failure --- tests/optimize/test_backtesting.py | 61 ++++++++++++++++++------------ 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 6912184aa..0b964c54a 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -90,28 +90,6 @@ def load_data_test(what, testdatadir): fill_missing=True)} -def simple_backtest(config, contour, mocker, testdatadir) -> None: - patch_exchange(mocker) - config['timeframe'] = '1m' - backtesting = Backtesting(config) - backtesting._set_strategy(backtesting.strategylist[0]) - - data = load_data_test(contour, testdatadir) - processed = backtesting.strategy.advise_all_indicators(data) - min_date, max_date = get_timerange(processed) - assert isinstance(processed, dict) - results = backtesting.backtest( - processed=processed, - start_date=min_date, - end_date=max_date, - max_open_trades=1, - position_stacking=False, - enable_protections=config.get('enable_protections', False), - ) - # results :: - return results - - # FIX: fixturize this? def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC'): data = history.load_data(datadir=datadir, timeframe='1m', pairs=[pair]) @@ -942,6 +920,7 @@ def test_backtest_dataprovider_analyzed_df(default_conf, fee, mocker, testdatadi def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatadir) -> None: # While this test IS a copy of test_backtest_pricecontours, it's needed to ensure # results do not carry-over to the next run, which is not given by using parametrize. + patch_exchange(mocker) default_conf['protections'] = [ { "method": "CooldownPeriod", @@ -949,6 +928,7 @@ def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatad }] default_conf['enable_protections'] = True + default_conf['timeframe'] = '1m' mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) mocker.patch("freqtrade.exchange.Exchange.get_max_pair_stake_amount", return_value=float('inf')) @@ -959,12 +939,27 @@ def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatad ['sine', 9], ['raise', 10], ] + backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) + # While entry-signals are unrealistic, running backtesting # over and over again should not cause different results for [contour, numres] in tests: # Debug output for random test failure print(f"{contour}, {numres}") - assert len(simple_backtest(default_conf, contour, mocker, testdatadir)['results']) == numres + data = load_data_test(contour, testdatadir) + processed = backtesting.strategy.advise_all_indicators(data) + min_date, max_date = get_timerange(processed) + assert isinstance(processed, dict) + results = backtesting.backtest( + processed=processed, + start_date=min_date, + end_date=max_date, + max_open_trades=1, + position_stacking=False, + enable_protections=default_conf.get('enable_protections', False), + ) + assert len(results['results']) == numres @pytest.mark.parametrize('protections,contour,expected', [ @@ -990,7 +985,25 @@ def test_backtest_pricecontours(default_conf, fee, mocker, testdatadir, mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) # While entry-signals are unrealistic, running backtesting # over and over again should not cause different results - assert len(simple_backtest(default_conf, contour, mocker, testdatadir)['results']) == expected + + patch_exchange(mocker) + default_conf['timeframe'] = '1m' + backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) + + data = load_data_test(contour, testdatadir) + processed = backtesting.strategy.advise_all_indicators(data) + min_date, max_date = get_timerange(processed) + assert isinstance(processed, dict) + results = backtesting.backtest( + processed=processed, + start_date=min_date, + end_date=max_date, + max_open_trades=1, + position_stacking=False, + enable_protections=default_conf.get('enable_protections', False), + ) + assert len(results['results']) == expected def test_backtest_clash_buy_sell(mocker, default_conf, testdatadir): From 05a5ae4fcfac1423a67b80411535a0b2ba07da27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Jul 2022 22:28:46 +0200 Subject: [PATCH 10/30] Update plotting to use entry/exit terminology --- freqtrade/plot/plotting.py | 22 +++++++++++----------- tests/test_plotting.py | 36 ++++++++++++++++++------------------ 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index a64281156..f8e95300a 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -255,18 +255,18 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: """ # Trades can be empty if trades is not None and len(trades) > 0: - # Create description for sell summarizing the trade + # Create description for exit summarizing the trade trades['desc'] = trades.apply( lambda row: f"{row['profit_ratio']:.2%}, " + (f"{row['enter_tag']}, " if row['enter_tag'] is not None else "") + f"{row['exit_reason']}, " + f"{row['trade_duration']} min", axis=1) - trade_buys = go.Scatter( + trade_entries = go.Scatter( x=trades["open_date"], y=trades["open_rate"], mode='markers', - name='Trade buy', + name='Trade entry', text=trades["desc"], marker=dict( symbol='circle-open', @@ -277,12 +277,12 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: ) ) - trade_sells = go.Scatter( + trade_exits = go.Scatter( x=trades.loc[trades['profit_ratio'] > 0, "close_date"], y=trades.loc[trades['profit_ratio'] > 0, "close_rate"], text=trades.loc[trades['profit_ratio'] > 0, "desc"], mode='markers', - name='Sell - Profit', + name='Exit - Profit', marker=dict( symbol='square-open', size=11, @@ -290,12 +290,12 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: color='green' ) ) - trade_sells_loss = go.Scatter( + trade_exits_loss = go.Scatter( x=trades.loc[trades['profit_ratio'] <= 0, "close_date"], y=trades.loc[trades['profit_ratio'] <= 0, "close_rate"], text=trades.loc[trades['profit_ratio'] <= 0, "desc"], mode='markers', - name='Sell - Loss', + name='Exit - Loss', marker=dict( symbol='square-open', size=11, @@ -303,9 +303,9 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: color='red' ) ) - fig.add_trace(trade_buys, 1, 1) - fig.add_trace(trade_sells, 1, 1) - fig.add_trace(trade_sells_loss, 1, 1) + fig.add_trace(trade_entries, 1, 1) + fig.add_trace(trade_exits, 1, 1) + fig.add_trace(trade_exits_loss, 1, 1) else: logger.warning("No trades found.") return fig @@ -444,7 +444,7 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra Generate the graph from the data generated by Backtesting or from DB Volume will always be ploted in row2, so Row 1 and 3 are to our disposal for custom indicators :param pair: Pair to Display on the graph - :param data: OHLCV DataFrame containing indicators and buy/sell signals + :param data: OHLCV DataFrame containing indicators and entry/exit signals :param trades: All trades created :param indicators1: List containing Main plot indicators :param indicators2: List containing Sub plot indicators diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 9ee7a75c6..52e96e477 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -72,7 +72,7 @@ def test_add_indicators(default_conf, testdatadir, caplog): strategy = StrategyResolver.load_strategy(default_conf) - # Generate buy/sell signals and indicators + # Generate entry/exit signals and indicators data = strategy.analyze_ticker(data, {'pair': pair}) fig = generate_empty_figure() @@ -113,7 +113,7 @@ def test_add_areas(default_conf, testdatadir, caplog): ind_plain = {"macd": {"fill_to": "macdhist"}} strategy = StrategyResolver.load_strategy(default_conf) - # Generate buy/sell signals and indicators + # Generate entry/exit signals and indicators data = strategy.analyze_ticker(data, {'pair': pair}) fig = generate_empty_figure() @@ -165,24 +165,24 @@ def test_plot_trades(testdatadir, caplog): fig = plot_trades(fig, trades) figure = fig1.layout.figure - # Check buys - color, should be in first graph, ... - trade_buy = find_trace_in_fig_data(figure.data, 'Trade buy') - assert isinstance(trade_buy, go.Scatter) - assert trade_buy.yaxis == 'y' - assert len(trades) == len(trade_buy.x) - assert trade_buy.marker.color == 'cyan' - assert trade_buy.marker.symbol == 'circle-open' - assert trade_buy.text[0] == '3.99%, buy_tag, roi, 15 min' + # Check entry - color, should be in first graph, ... + trade_entries = find_trace_in_fig_data(figure.data, 'Trade entry') + assert isinstance(trade_entries, go.Scatter) + assert trade_entries.yaxis == 'y' + assert len(trades) == len(trade_entries.x) + assert trade_entries.marker.color == 'cyan' + assert trade_entries.marker.symbol == 'circle-open' + assert trade_entries.text[0] == '3.99%, buy_tag, roi, 15 min' - trade_sell = find_trace_in_fig_data(figure.data, 'Sell - Profit') - assert isinstance(trade_sell, go.Scatter) - assert trade_sell.yaxis == 'y' - assert len(trades.loc[trades['profit_ratio'] > 0]) == len(trade_sell.x) - assert trade_sell.marker.color == 'green' - assert trade_sell.marker.symbol == 'square-open' - assert trade_sell.text[0] == '3.99%, buy_tag, roi, 15 min' + trade_exit = find_trace_in_fig_data(figure.data, 'Exit - Profit') + assert isinstance(trade_exit, go.Scatter) + assert trade_exit.yaxis == 'y' + assert len(trades.loc[trades['profit_ratio'] > 0]) == len(trade_exit.x) + assert trade_exit.marker.color == 'green' + assert trade_exit.marker.symbol == 'square-open' + assert trade_exit.text[0] == '3.99%, buy_tag, roi, 15 min' - trade_sell_loss = find_trace_in_fig_data(figure.data, 'Sell - Loss') + trade_sell_loss = find_trace_in_fig_data(figure.data, 'Exit - Loss') assert isinstance(trade_sell_loss, go.Scatter) assert trade_sell_loss.yaxis == 'y' assert len(trades.loc[trades['profit_ratio'] <= 0]) == len(trade_sell_loss.x) From 46be1b8778c9729f16651c2b8f00ac78dd98de98 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 17 Jul 2022 07:21:42 +0200 Subject: [PATCH 11/30] Version bump ccxt to 1.90.88 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2bb3b4b75..49121a9da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.23.1 pandas==1.4.3 pandas-ta==0.3.14b -ccxt==1.90.47 +ccxt==1.90.88 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.4 aiohttp==3.8.1 From 2c6fb617a6574ac9342fdbaa937938e581de525a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 03:01:23 +0000 Subject: [PATCH 12/30] Bump jsonschema from 4.6.2 to 4.7.2 Bumps [jsonschema](https://github.com/python-jsonschema/jsonschema) from 4.6.2 to 4.7.2. - [Release notes](https://github.com/python-jsonschema/jsonschema/releases) - [Changelog](https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-jsonschema/jsonschema/compare/v4.6.2...v4.7.2) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 49121a9da..78a327988 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ arrow==1.2.2 cachetools==4.2.2 requests==2.28.1 urllib3==1.26.10 -jsonschema==4.6.2 +jsonschema==4.7.2 TA-Lib==0.4.24 technical==1.3.0 tabulate==0.8.10 From 5f820ab0a64a0ebe0152c74901cb890d8a8a0cd9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 03:01:26 +0000 Subject: [PATCH 13/30] Bump pytest-asyncio from 0.18.3 to 0.19.0 Bumps [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) from 0.18.3 to 0.19.0. - [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases) - [Changelog](https://github.com/pytest-dev/pytest-asyncio/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.18.3...v0.19.0) --- updated-dependencies: - dependency-name: pytest-asyncio dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f2f77c2ba..ac7f4f619 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ flake8-tidy-imports==4.8.0 mypy==0.961 pre-commit==2.20.0 pytest==7.1.2 -pytest-asyncio==0.18.3 +pytest-asyncio==0.19.0 pytest-cov==3.0.0 pytest-mock==3.8.2 pytest-random-order==1.0.4 From cb63d5e3dfe9dc9b8275098ba8b7cf6dc446ac2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 03:01:31 +0000 Subject: [PATCH 14/30] Bump fastapi from 0.78.0 to 0.79.0 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.78.0 to 0.79.0. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.78.0...0.79.0) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 49121a9da..18c656bfb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ orjson==3.7.7 sdnotify==0.3.2 # API Server -fastapi==0.78.0 +fastapi==0.79.0 uvicorn==0.18.2 pyjwt==2.4.0 aiofiles==0.8.0 From f07ad7aa875a3c0c53de3c70b73330a90676d6c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 03:01:40 +0000 Subject: [PATCH 15/30] Bump ccxt from 1.90.88 to 1.90.89 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.90.88 to 1.90.89. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.90.88...1.90.89) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 49121a9da..9c9973ee8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.23.1 pandas==1.4.3 pandas-ta==0.3.14b -ccxt==1.90.88 +ccxt==1.90.89 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.4 aiohttp==3.8.1 From d2ef248781718ce27e3c59cad6539ba91adb01b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 03:01:43 +0000 Subject: [PATCH 16/30] Bump markdown from 3.3.7 to 3.4.1 Bumps [markdown](https://github.com/Python-Markdown/markdown) from 3.3.7 to 3.4.1. - [Release notes](https://github.com/Python-Markdown/markdown/releases) - [Commits](https://github.com/Python-Markdown/markdown/compare/3.3.7...3.4.1) --- updated-dependencies: - dependency-name: markdown dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 22d92c65d..a5fe8b921 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ -markdown==3.3.7 +markdown==3.4.1 mkdocs==1.3.0 mkdocs-material==8.3.9 mdx_truly_sane_lists==1.2 From ea523136fc29cdf62a14c59e85edfe089ac03b1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 03:01:49 +0000 Subject: [PATCH 17/30] Bump types-requests from 2.28.0 to 2.28.1 Bumps [types-requests](https://github.com/python/typeshed) from 2.28.0 to 2.28.1. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f2f77c2ba..435cb9b1c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -24,6 +24,6 @@ nbconvert==6.5.0 # mypy types types-cachetools==5.2.1 types-filelock==3.2.7 -types-requests==2.28.0 +types-requests==2.28.1 types-tabulate==0.8.11 types-python-dateutil==2.8.18 From 0daa9d3e57bfa09f98c17d14258dd9fad32803dd Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 18 Jul 2022 07:56:41 +0200 Subject: [PATCH 18/30] Bump types-requests in pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 59e7f6894..a23181c37 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: additional_dependencies: - types-cachetools==5.2.1 - types-filelock==3.2.7 - - types-requests==2.28.0 + - types-requests==2.28.1 - types-tabulate==0.8.11 - types-python-dateutil==2.8.18 # stages: [push] From 75e190ff1ddfecfd1796d99dbde4a060a18e96b7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 18 Jul 2022 20:15:07 +0200 Subject: [PATCH 19/30] Update sell-test without filled buy order --- tests/test_freqtradebot.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 4963e2b0a..e431e7ac3 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2060,8 +2060,9 @@ def test_update_trade_state_orderexception(mocker, default_conf_usdt, caplog) -> @pytest.mark.parametrize("is_short", [False, True]) def test_update_trade_state_sell( - default_conf_usdt, trades_for_order, limit_order_open, limit_order, is_short, mocker, + default_conf_usdt, trades_for_order, limit_order_open, limit_order, is_short, mocker ): + buy_order = limit_order[entry_side(is_short)] open_order = limit_order_open[exit_side(is_short)] l_order = limit_order[exit_side(is_short)] mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) @@ -2088,6 +2089,9 @@ def test_update_trade_state_sell( leverage=1, is_short=is_short, ) + order = Order.parse_from_ccxt_object(buy_order, 'LTC/ETH', entry_side(is_short)) + trade.orders.append(order) + order = Order.parse_from_ccxt_object(open_order, 'LTC/ETH', exit_side(is_short)) trade.orders.append(order) assert order.status == 'open' @@ -2787,6 +2791,7 @@ def test_manage_open_orders_partial( rpc_mock = patch_RPCManager(mocker) open_trade.is_short = is_short open_trade.leverage = leverage + open_trade.orders[0].ft_order_side = 'sell' if is_short else 'buy' limit_buy_order_old_partial['id'] = open_trade.open_order_id limit_buy_order_old_partial['side'] = 'sell' if is_short else 'buy' limit_buy_canceled = deepcopy(limit_buy_order_old_partial) @@ -2872,6 +2877,7 @@ def test_manage_open_orders_partial_except( limit_buy_order_old_partial_canceled, mocker ) -> None: open_trade.is_short = is_short + open_trade.orders[0].ft_order_side = 'sell' if is_short else 'buy' rpc_mock = patch_RPCManager(mocker) limit_buy_order_old_partial_canceled['id'] = open_trade.open_order_id limit_buy_order_old_partial['id'] = open_trade.open_order_id @@ -3626,7 +3632,7 @@ def test_execute_trade_exit_market_order( 'freqtrade.exchange.Exchange', fetch_ticker=ticker_usdt, get_fee=fee, - _is_dry_limit_order_filled=MagicMock(return_value=False), + _is_dry_limit_order_filled=MagicMock(return_value=True), ) patch_whitelist(mocker, default_conf_usdt) freqtrade = FreqtradeBot(default_conf_usdt) @@ -3642,7 +3648,8 @@ def test_execute_trade_exit_market_order( # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker_usdt_sell_up + fetch_ticker=ticker_usdt_sell_up, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) freqtrade.config['order_types']['exit'] = 'market' @@ -3655,7 +3662,7 @@ def test_execute_trade_exit_market_order( assert not trade.is_open assert trade.close_profit == profit_ratio - assert rpc_mock.call_count == 3 + assert rpc_mock.call_count == 4 last_msg = rpc_mock.call_args_list[-2][0][0] assert { 'type': RPCMessageType.EXIT, From b609dbcd8648937ed00785b0cab32faca1512859 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 19 Jul 2022 19:51:03 +0200 Subject: [PATCH 20/30] Update mdx_truly_sane_lists to be compatible with markdown again --- .gitignore | 2 ++ docs/requirements-docs.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 97f77f779..d6cec5225 100644 --- a/.gitignore +++ b/.gitignore @@ -80,6 +80,8 @@ instance/ # Sphinx documentation docs/_build/ +# Mkdocs documentation +site/ # PyBuilder target/ diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index a5fe8b921..a07f4f944 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.4.1 mkdocs==1.3.0 mkdocs-material==8.3.9 -mdx_truly_sane_lists==1.2 +mdx_truly_sane_lists==1.3 pymdown-extensions==9.5 jinja2==3.1.2 From 32c3f62934f97520490be1b009a5d0f5dc7dab4f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 22 Jul 2022 19:45:00 +0200 Subject: [PATCH 21/30] Fix documentation typo closes #7115 --- docs/stoploss.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index 6ddb485a4..249c40109 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -175,8 +175,8 @@ Before this, `stoploss` is used for the trailing stoploss. * assuming the asset now increases to 102$ * the stoploss will now be at 91.8$ - 10% below the highest observed rate * assuming the asset now increases to 103.5$ (above the offset configured) - * the stop loss will now be -2% of 103$ = 101.42$ - * now the asset drops in value to 102\$, the stop loss will still be 101.42$ and would trigger once price breaks below 101.42$ + * the stop loss will now be -2% of 103.5$ = 101.43$ + * now the asset drops in value to 102\$, the stop loss will still be 101.43$ and would trigger once price breaks below 101.43$ ### Trailing stop loss only once the trade has reached a certain offset From e97468964a46bc7464928a1626fa60f7b946ffbc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 08:42:20 +0200 Subject: [PATCH 22/30] Add support for --timeframe-detail in hyperopt fix #7070 --- docs/hyperopt.md | 23 ++++++++++++++++------- freqtrade/commands/arguments.py | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 55fe8f008..c9ec30056 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -40,13 +40,15 @@ pip install -r requirements-hyperopt.txt ``` usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] - [-i TIMEFRAME] [--timerange TIMERANGE] + [--recursive-strategy-search] [-i TIMEFRAME] + [--timerange TIMERANGE] [--data-format-ohlcv {json,jsongz,hdf5}] [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [-p PAIRS [PAIRS ...]] [--hyperopt-path PATH] [--eps] [--dmmp] [--enable-protections] - [--dry-run-wallet DRY_RUN_WALLET] [-e INT] + [--dry-run-wallet DRY_RUN_WALLET] + [--timeframe-detail TIMEFRAME_DETAIL] [-e INT] [--spaces {all,buy,sell,roi,stoploss,trailing,protection,default} [{all,buy,sell,roi,stoploss,trailing,protection,default} ...]] [--print-all] [--no-color] [--print-json] [-j JOBS] [--random-state INT] [--min-trades INT] @@ -89,6 +91,9 @@ optional arguments: --dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET Starting balance, used for backtesting / hyperopt and dry-runs. + --timeframe-detail TIMEFRAME_DETAIL + Specify detail timeframe for backtesting (`1m`, `5m`, + `30m`, `1h`, `1d`). -e INT, --epochs INT Specify number of epochs (default: 100). --spaces {all,buy,sell,roi,stoploss,trailing,protection,default} [{all,buy,sell,roi,stoploss,trailing,protection,default} ...] Specify which parameters to hyperopt. Space-separated @@ -146,7 +151,9 @@ Strategy arguments: Specify strategy class name which will be used by the bot. --strategy-path PATH Specify additional strategy lookup path. - + --recursive-strategy-search + Recursively search for a strategy in the strategies + folder. ``` ### Hyperopt checklist @@ -867,10 +874,12 @@ You can also enable position stacking in the configuration file by explicitly se As hyperopt consumes a lot of memory (the complete data needs to be in memory once per parallel backtesting process), it's likely that you run into "out of memory" errors. To combat these, you have multiple options: -* reduce the amount of pairs -* reduce the timerange used (`--timerange `) -* reduce the number of parallel processes (`-j `) -* Increase the memory of your machine +* Reduce the amount of pairs. +* Reduce the timerange used (`--timerange `). +* Avoid using `--timeframe-detail` (this loads a lot of additional data into memory). +* Reduce the number of parallel processes (`-j `). +* Increase the memory of your machine. + ## The objective has been evaluated at this point before. diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 1e3e2845a..fc3eda14d 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -28,7 +28,7 @@ ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_pos ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", "position_stacking", "use_max_market_positions", - "enable_protections", "dry_run_wallet", + "enable_protections", "dry_run_wallet", "timeframe_detail", "epochs", "spaces", "print_all", "print_colorized", "print_json", "hyperopt_jobs", "hyperopt_random_state", "hyperopt_min_trades", From 5c4f60f376452783674819a2b2e02e0e6ce8acad Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 09:11:22 +0200 Subject: [PATCH 23/30] Improve configuration table formatting and ordering --- docs/configuration.md | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0f3069478..412571674 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -116,6 +116,9 @@ This is similar to using multiple `--config` parameters, but simpler in usage as The table below will list all configuration parameters available. Freqtrade can also load many options via command line (CLI) arguments (check out the commands `--help` output for details). + +### Configuration option prevalence + The prevalence for all Options is as follows: - CLI arguments override any other option @@ -123,6 +126,8 @@ The prevalence for all Options is as follows: - Configuration files are used in sequence (the last file wins) and override Strategy configurations. - Strategy configurations are only used if they are not set via configuration or command-line arguments. These options are marked with [Strategy Override](#parameters-in-the-strategy) in the below table. +### Parameters table + Mandatory parameters are marked as **Required**, which means that they are required to be set in one of the possible ways. | Parameter | Description | @@ -135,7 +140,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `amend_last_stake_amount` | Use reduced last stake amount if necessary. [More information below](#configuring-amount-per-trade).
*Defaults to `false`.*
**Datatype:** Boolean | `last_stake_amount_min_ratio` | Defines minimum stake amount that has to be left and executed. Applies only to the last stake amount when it's amended to a reduced value (i.e. if `amend_last_stake_amount` is set to `true`). [More information below](#configuring-amount-per-trade).
*Defaults to `0.5`.*
**Datatype:** Float (as ratio) | `amount_reserve_percent` | Reserve some amount in min pair stake amount. The bot will reserve `amount_reserve_percent` + stoploss value when calculating min pair stake amount in order to avoid possible trade refusals.
*Defaults to `0.05` (5%).*
**Datatype:** Positive Float as ratio. -| `timeframe` | The timeframe to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** String +| `timeframe` | The timeframe to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). Usually missing in configuration, and specified in the strategy. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** String | `fiat_display_currency` | Fiat currency used to show your profits. [More information below](#what-values-can-be-used-for-fiat_display_currency).
**Datatype:** String | `dry_run` | **Required.** Define if the bot must be in Dry Run or production mode.
*Defaults to `true`.*
**Datatype:** Boolean | `dry_run_wallet` | Define the starting amount in stake currency for the simulated wallet used by the bot running in Dry Run mode.
*Defaults to `1000`.*
**Datatype:** Float @@ -148,13 +153,16 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md#trailing-stop-loss-only-once-the-trade-has-reached-a-certain-offset). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0` (no offset).*
**Datatype:** Float | `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `fee` | Fee used during backtesting / dry-runs. Should normally not be configured, which has freqtrade fall back to the exchange default fee. Set as ratio (e.g. 0.001 = 0.1%). Fee is applied twice for each trade, once when buying, once when selling.
**Datatype:** Float (as ratio) +| `futures_funding_rate` | User-specified funding rate to be used when historical funding rates are not available from the exchange. This does not overwrite real historical rates. It is recommended that this be set to 0 unless you are testing a specific coin and you understand how the funding rate will affect freqtrade's profit calculations. [More information here](leverage.md#unavailable-funding-rates)
*Defaults to None.*
**Datatype:** Float | `trading_mode` | Specifies if you want to trade regularly, trade with leverage, or trade contracts whose prices are derived from matching cryptocurrency prices. [leverage documentation](leverage.md).
*Defaults to `"spot"`.*
**Datatype:** String | `margin_mode` | When trading with leverage, this determines if the collateral owned by the trader will be shared or isolated to each trading pair [leverage documentation](leverage.md).
**Datatype:** String | `liquidation_buffer` | A ratio specifying how large of a safety net to place between the liquidation price and the stoploss to prevent a position from reaching the liquidation price [leverage documentation](leverage.md).
*Defaults to `0.05`.*
**Datatype:** Float +| | **Unfilled timeout** | `unfilledtimeout.entry` | **Required.** How long (in minutes or seconds) the bot will wait for an unfilled entry order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer | `unfilledtimeout.exit` | **Required.** How long (in minutes or seconds) the bot will wait for an unfilled exit order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer | `unfilledtimeout.unit` | Unit to use in unfilledtimeout setting. Note: If you set unfilledtimeout.unit to "seconds", "internals.process_throttle_secs" must be inferior or equal to timeout [Strategy Override](#parameters-in-the-strategy).
*Defaults to `minutes`.*
**Datatype:** String | `unfilledtimeout.exit_timeout_count` | How many times can exit orders time out. Once this number of timeouts is reached, an emergency exit is triggered. 0 to disable and allow unlimited order cancels. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0`.*
**Datatype:** Integer +| | **Pricing** | `entry_pricing.price_side` | Select the side of the spread the bot should look at to get the entry rate. [More information below](#buy-price-side).
*Defaults to `same`.*
**Datatype:** String (either `ask`, `bid`, `same` or `other`). | `entry_pricing.price_last_balance` | **Required.** Interpolate the bidding price. More information [below](#entry-price-without-orderbook-enabled). | `entry_pricing.use_order_book` | Enable entering using the rates in [Order Book Entry](#entry-price-with-orderbook-enabled).
*Defaults to `True`.*
**Datatype:** Boolean @@ -165,6 +173,8 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exit_pricing.price_last_balance` | Interpolate the exiting price. More information [below](#exit-price-without-orderbook-enabled). | `exit_pricing.use_order_book` | Enable exiting of open trades using [Order Book Exit](#exit-price-with-orderbook-enabled).
*Defaults to `True`.*
**Datatype:** Boolean | `exit_pricing.order_book_top` | Bot will use the top N rate in Order Book "price_side" to exit. I.e. a value of 2 will allow the bot to pick the 2nd ask rate in [Order Book Exit](#exit-price-with-orderbook-enabled)
*Defaults to `1`.*
**Datatype:** Positive Integer +| `custom_price_max_distance_ratio` | Configure maximum distance ratio between current and custom entry or exit price.
*Defaults to `0.02` 2%).*
**Datatype:** Positive float +| | **TODO** | `use_exit_signal` | Use exit signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
**Datatype:** Boolean | `exit_profit_only` | Wait until the bot reaches `exit_profit_offset` before taking an exit decision. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `exit_profit_offset` | Exit-signal is only active above this value. Only active in combination with `exit_profit_only=True`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0`.*
**Datatype:** Float (as ratio) @@ -172,8 +182,9 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `ignore_buying_expired_candle_after` | Specifies the number of seconds until a buy signal is no longer used.
**Datatype:** Integer | `order_types` | Configure order-types depending on the action (`"entry"`, `"exit"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict | `order_time_in_force` | Configure time in force for entry and exit orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict -| `custom_price_max_distance_ratio` | Configure maximum distance ratio between current and custom entry or exit price.
*Defaults to `0.02` 2%).*
**Datatype:** Positive float -| `recursive_strategy_search` | Set to `true` to recursively search sub-directories inside `user_data/strategies` for a strategy.
**Datatype:** Boolean +| `position_adjustment_enable` | Enables the strategy to use position adjustments (additional buys or sells). [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean +| `max_entry_position_adjustment` | Maximum additional order(s) for each open trade on top of the first entry Order. Set it to `-1` for unlimited additional orders. [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `-1`.*
**Datatype:** Positive Integer or -1 +| | **Exchange** | `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename).
**Datatype:** String | `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.
**Datatype:** Boolean | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String @@ -190,14 +201,19 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.skip_open_order_update` | Skips open order updates on startup should the exchange cause problems. Only relevant in live conditions.
*Defaults to `false`
**Datatype:** Boolean | `exchange.unknown_fee_rate` | Fallback value to use when calculating trading fees. This can be useful for exchanges which have fees in non-tradable currencies. The value provided here will be multiplied with the "fee cost".
*Defaults to `None`
**Datatype:** float | `exchange.log_responses` | Log relevant exchange responses. For debug mode only - use with care.
*Defaults to `false`
**Datatype:** Boolean -| `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
**Datatype:** Boolean +| | **Plugins** +| `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation of all possible configuration options. | `pairlists` | Define one or more pairlists to be used. [More information](plugins.md#pairlists-and-pairlist-handlers).
*Defaults to `StaticPairList`.*
**Datatype:** List of Dicts | `protections` | Define one or more protections to be used. [More information](plugins.md#protections).
**Datatype:** List of Dicts +| | **Telegram** | `telegram.enabled` | Enable the usage of Telegram.
**Datatype:** Boolean | `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `telegram.balance_dust_level` | Dust-level (in stake currency) - currencies with a balance below this will not be shown by `/balance`.
**Datatype:** float +| `telegram.reload` | Allow "reload" buttons on telegram messages.
*Defaults to `True`.
**Datatype:** boolean +| `telegram.notification_settings.*` | Detailed notification settings. Refer to the [telegram documentation](telegram-usage.md) for details.
**Datatype:** dictionary +| | **Webhook** | `webhook.enabled` | Enable usage of Webhook notifications
**Datatype:** Boolean | `webhook.url` | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String | `webhook.webhookentry` | Payload to send on entry. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String @@ -207,6 +223,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `webhook.webhookexitcancel` | Payload to send on exit order cancel. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String | `webhook.webhookexitfill` | Payload to send on exit order filled. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String | `webhook.webhookstatus` | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String +| | **Rest API / FreqUI** | `api_server.enabled` | Enable usage of API Server. See the [API Server documentation](rest-api.md) for more details.
**Datatype:** Boolean | `api_server.listen_ip_address` | Bind IP address. See the [API Server documentation](rest-api.md) for more details.
**Datatype:** IPv4 | `api_server.listen_port` | Bind Port. See the [API Server documentation](rest-api.md) for more details.
**Datatype:** Integer between 1024 and 65535 @@ -214,23 +231,22 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `api_server.username` | Username for API server. See the [API Server documentation](rest-api.md) for more details.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `api_server.password` | Password for API server. See the [API Server documentation](rest-api.md) for more details.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `bot_name` | Name of the bot. Passed via API to a client - can be shown to distinguish / name bots.
*Defaults to `freqtrade`*
**Datatype:** String -| `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite:///tradesv3.dryrun.sqlite` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
**Datatype:** String, SQLAlchemy connect string +| | **Other** | `initial_state` | Defines the initial application state. If set to stopped, then the bot has to be explicitly started via `/start` RPC command.
*Defaults to `stopped`.*
**Datatype:** Enum, either `stopped` or `running` | `force_entry_enable` | Enables the RPC Commands to force a Trade entry. More information below.
**Datatype:** Boolean | `disable_dataframe_checks` | Disable checking the OHLCV dataframe returned from the strategy methods for correctness. Only use when intentionally changing the dataframe and understand what you are doing. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `False`*.
**Datatype:** Boolean -| `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`.
**Datatype:** ClassName -| `strategy_path` | Adds an additional strategy lookup path (must be a directory).
**Datatype:** String | `internals.process_throttle_secs` | Set the process throttle, or minimum loop duration for one bot iteration loop. Value in second.
*Defaults to `5` seconds.*
**Datatype:** Positive Integer | `internals.heartbeat_interval` | Print heartbeat message every N seconds. Set to 0 to disable heartbeat messages.
*Defaults to `60` seconds.*
**Datatype:** Positive Integer or 0 | `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
**Datatype:** Boolean -| `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file.
**Datatype:** String +| `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`.
**Datatype:** ClassName +| `strategy_path` | Adds an additional strategy lookup path (must be a directory).
**Datatype:** String +| `recursive_strategy_search` | Set to `true` to recursively search sub-directories inside `user_data/strategies` for a strategy.
**Datatype:** Boolean | `user_data_dir` | Directory containing user data.
*Defaults to `./user_data/`*.
**Datatype:** String +| `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite:///tradesv3.dryrun.sqlite` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
**Datatype:** String, SQLAlchemy connect string +| `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file.
**Datatype:** String | `add_config_files` | Additional config files. These files will be loaded and merged with the current config file. The files are resolved relative to the initial file.
*Defaults to `[]`*.
**Datatype:** List of strings | `dataformat_ohlcv` | Data format to use to store historical candle (OHLCV) data.
*Defaults to `json`*.
**Datatype:** String | `dataformat_trades` | Data format to use to store historical trades data.
*Defaults to `jsongz`*.
**Datatype:** String -| `position_adjustment_enable` | Enables the strategy to use position adjustments (additional buys or sells). [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean -| `max_entry_position_adjustment` | Maximum additional order(s) for each open trade on top of the first entry Order. Set it to `-1` for unlimited additional orders. [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `-1`.*
**Datatype:** Positive Integer or -1 -| `futures_funding_rate` | User-specified funding rate to be used when historical funding rates are not available from the exchange. This does not overwrite real historical rates. It is recommended that this be set to 0 unless you are testing a specific coin and you understand how the funding rate will affect freqtrade's profit calculations. [More information here](leverage.md#unavailable-funding-rates)
*Defaults to None.*
**Datatype:** Float ### Parameters in the strategy From a02d02ac1251a1525545ee5fa3ba87fb61ac5899 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 14:43:52 +0200 Subject: [PATCH 24/30] Enhance protections tests to have orders in mock trade --- tests/plugins/test_protections.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 172e1f077..3c333200c 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -6,6 +6,7 @@ import pytest from freqtrade import constants from freqtrade.enums import ExitType from freqtrade.persistence import PairLocks, Trade +from freqtrade.persistence.trade_model import Order from freqtrade.plugins.protectionmanager import ProtectionManager from tests.conftest import get_patched_freqtradebot, log_has_re @@ -30,7 +31,37 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool, amount=0.01 / open_rate, exchange='binance', is_short=is_short, + leverage=1, ) + + trade.orders.append(Order( + ft_order_side=trade.entry_side, + order_id=f'{pair}-{trade.entry_side}-{trade.open_date}', + ft_pair=pair, + amount=trade.amount, + filled=trade.amount, + remaining=0, + price=open_rate, + average=open_rate, + status="closed", + order_type="market", + side=trade.entry_side, + )) + if not is_open: + trade.orders.append(Order( + ft_order_side=trade.exit_side, + order_id=f'{pair}-{trade.exit_side}-{trade.close_date}', + ft_pair=pair, + amount=trade.amount, + filled=trade.amount, + remaining=0, + price=open_rate * (2 - profit_rate if is_short else profit_rate), + average=open_rate * (2 - profit_rate if is_short else profit_rate), + status="closed", + order_type="market", + side=trade.exit_side, + )) + trade.recalc_open_trade_value() if not is_open: trade.close(open_rate * (2 - profit_rate if is_short else profit_rate)) From 80845807e113527fcbf2f716db62a4beb2ddae89 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 15:14:38 +0200 Subject: [PATCH 25/30] Improve some test resiliance --- tests/test_freqtradebot.py | 14 ++++++++------ tests/test_persistence.py | 8 ++++++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index e431e7ac3..a4b10fbcd 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2139,8 +2139,6 @@ def test_handle_trade( assert trade time.sleep(0.01) # Race condition fix - oobj = Order.parse_from_ccxt_object(enter_order, enter_order['symbol'], entry_side(is_short)) - trade.update_trade(oobj) assert trade.is_open is True freqtrade.wallets.update() @@ -2150,11 +2148,15 @@ def test_handle_trade( assert trade.open_order_id == exit_order['id'] # Simulate fulfilled LIMIT_SELL order for trade - oobj = Order.parse_from_ccxt_object(exit_order, exit_order['symbol'], exit_side(is_short)) - trade.update_trade(oobj) + trade.orders[-1].ft_is_open = False + trade.orders[-1].status = 'closed' + trade.orders[-1].filled = trade.orders[-1].remaining + trade.orders[-1].remaining = 0.0 - assert trade.close_rate == 2.0 if is_short else 2.2 - assert trade.close_profit == close_profit + trade.update_trade(trade.orders[-1]) + + assert trade.close_rate == (2.0 if is_short else 2.2) + assert pytest.approx(trade.close_profit) == close_profit assert trade.calc_profit(trade.close_rate) == 5.685 assert trade.close_date is not None assert trade.exit_reason == 'sell_signal1' diff --git a/tests/test_persistence.py b/tests/test_persistence.py index a09711048..529f6dc31 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -481,6 +481,7 @@ def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_ trade.open_order_id = 'something' oobj = Order.parse_from_ccxt_object(enter_order, 'ADA/USDT', entry_side) + trade.orders.append(oobj) trade.update_trade(oobj) assert trade.open_order_id is None assert trade.open_rate == open_rate @@ -496,11 +497,12 @@ def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_ trade.open_order_id = 'something' time_machine.move_to("2022-03-31 21:45:05 +00:00") oobj = Order.parse_from_ccxt_object(exit_order, 'ADA/USDT', exit_side) + trade.orders.append(oobj) trade.update_trade(oobj) assert trade.open_order_id is None assert trade.close_rate == close_rate - assert trade.close_profit == profit + assert pytest.approx(trade.close_profit) == profit assert trade.close_date is not None assert log_has_re(f"LIMIT_{exit_side.upper()} has been fulfilled for " r"Trade\(id=2, pair=ADA/USDT, amount=30.00000000, " @@ -529,6 +531,7 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, trade.open_order_id = 'something' oobj = Order.parse_from_ccxt_object(market_buy_order_usdt, 'ADA/USDT', 'buy') + trade.orders.append(oobj) trade.update_trade(oobj) assert trade.open_order_id is None assert trade.open_rate == 2.0 @@ -543,10 +546,11 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, trade.is_open = True trade.open_order_id = 'something' oobj = Order.parse_from_ccxt_object(market_sell_order_usdt, 'ADA/USDT', 'sell') + trade.orders.append(oobj) trade.update_trade(oobj) assert trade.open_order_id is None assert trade.close_rate == 2.2 - assert trade.close_profit == round(0.0945137157107232, 8) + assert pytest.approx(trade.close_profit) == 0.094513715710723 assert trade.close_date is not None assert log_has_re(r"MARKET_SELL has been fulfilled for Trade\(id=1, " r"pair=ADA/USDT, amount=30.00000000, is_short=False, leverage=1.0, " From 24a786bedd7e83a9e68e99041f6dd4d89fd018fc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 15:23:24 +0200 Subject: [PATCH 26/30] Update rpc test to contain sell order --- tests/rpc/test_rpc_telegram.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 91ee92fd7..f69b7e878 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -686,6 +686,7 @@ def test_profit_handle(default_conf_usdt, update, ticker_usdt, ticker_sell_up, f # Simulate fulfilled LIMIT_SELL order for trade oobj = Order.parse_from_ccxt_object( limit_sell_order_usdt, limit_sell_order_usdt['symbol'], 'sell') + trade.orders.append(oobj) trade.update_trade(oobj) trade.close_date = datetime.now(timezone.utc) @@ -707,7 +708,7 @@ def test_profit_handle(default_conf_usdt, update, ticker_usdt, ticker_sell_up, f assert '*Best Performing:* `ETH/USDT: 9.45%`' in msg_mock.call_args_list[-1][0][0] assert '*Max Drawdown:*' in msg_mock.call_args_list[-1][0][0] assert '*Profit factor:*' in msg_mock.call_args_list[-1][0][0] - assert '*Trading volume:* `60 USDT`' in msg_mock.call_args_list[-1][0][0] + assert '*Trading volume:* `126 USDT`' in msg_mock.call_args_list[-1][0][0] @pytest.mark.parametrize('is_short', [True, False]) From 7682c9ace70d6a6ad1e6a5f0c283d8dad067790c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 15:27:52 +0200 Subject: [PATCH 27/30] Update trade_close test to include orders --- tests/test_persistence.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 529f6dc31..838c4c22a 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -628,14 +628,41 @@ def test_trade_close(limit_buy_order_usdt, limit_sell_order_usdt, fee): open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=10), interest_rate=0.0005, exchange='binance', - trading_mode=margin + trading_mode=margin, + leverage=1.0, ) + trade.orders.append(Order( + ft_order_side=trade.entry_side, + order_id=f'{trade.pair}-{trade.entry_side}-{trade.open_date}', + ft_pair=trade.pair, + amount=trade.amount, + filled=trade.amount, + remaining=0, + price=trade.open_rate, + average=trade.open_rate, + status="closed", + order_type="limit", + side=trade.entry_side, + )) + trade.orders.append(Order( + ft_order_side=trade.exit_side, + order_id=f'{trade.pair}-{trade.exit_side}-{trade.open_date}', + ft_pair=trade.pair, + amount=trade.amount, + filled=trade.amount, + remaining=0, + price=2.2, + average=2.2, + status="closed", + order_type="limit", + side=trade.exit_side, + )) assert trade.close_profit is None assert trade.close_date is None assert trade.is_open is True trade.close(2.2) assert trade.is_open is False - assert trade.close_profit == round(0.0945137157107232, 8) + assert pytest.approx(trade.close_profit) == 0.094513715 assert trade.close_date is not None new_date = arrow.Arrow(2020, 2, 2, 15, 6, 1).datetime, From 2eb1d18c2a8f788fa47826ae8fca45795bcba56d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 19:56:38 +0200 Subject: [PATCH 28/30] Don't load leverage tiers when not necessary --- freqtrade/exchange/exchange.py | 5 +++-- freqtrade/freqtradebot.py | 3 ++- freqtrade/optimize/backtesting.py | 3 ++- freqtrade/resolvers/exchange_resolver.py | 13 +++++++++---- freqtrade/rpc/api_server/deps.py | 2 +- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index a430cdac5..11e37b953 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -88,7 +88,8 @@ class Exchange: # TradingMode.SPOT always supported and not required in this list ] - def __init__(self, config: Dict[str, Any], validate: bool = True) -> None: + def __init__(self, config: Dict[str, Any], validate: bool = True, + load_leverage_tiers: bool = False) -> None: """ Initializes this module with the given config, it does basic validation whether the specified exchange and pairs are valid. @@ -186,7 +187,7 @@ class Exchange: self.markets_refresh_interval: int = exchange_config.get( "markets_refresh_interval", 60) * 60 - if self.trading_mode != TradingMode.SPOT: + if self.trading_mode != TradingMode.SPOT and load_leverage_tiers: self.fill_leverage_tiers() self.additional_exchange_init() diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 2007f9b4e..43608cae7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -65,7 +65,8 @@ class FreqtradeBot(LoggingMixin): # Check config consistency here since strategies can set certain options validate_config_consistency(config) - self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) + self.exchange = ExchangeResolver.load_exchange( + self.config['exchange']['name'], self.config, load_leverage_tiers=True) init_db(self.config['db_url']) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index da28a8d93..4d16dc0f1 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -84,7 +84,8 @@ class Backtesting: self.processed_dfs: Dict[str, Dict] = {} self._exchange_name = self.config['exchange']['name'] - self.exchange = ExchangeResolver.load_exchange(self._exchange_name, self.config) + self.exchange = ExchangeResolver.load_exchange( + self._exchange_name, self.config, load_leverage_tiers=True) self.dataprovider = DataProvider(self.config, self.exchange) if self.config.get('strategy_list'): diff --git a/freqtrade/resolvers/exchange_resolver.py b/freqtrade/resolvers/exchange_resolver.py index 4dfbf445b..a2f572ff2 100644 --- a/freqtrade/resolvers/exchange_resolver.py +++ b/freqtrade/resolvers/exchange_resolver.py @@ -18,7 +18,8 @@ class ExchangeResolver(IResolver): object_type = Exchange @staticmethod - def load_exchange(exchange_name: str, config: dict, validate: bool = True) -> Exchange: + def load_exchange(exchange_name: str, config: dict, validate: bool = True, + load_leverage_tiers: bool = False) -> Exchange: """ Load the custom class from config parameter :param exchange_name: name of the Exchange to load @@ -29,9 +30,13 @@ class ExchangeResolver(IResolver): exchange_name = exchange_name.title() exchange = None try: - exchange = ExchangeResolver._load_exchange(exchange_name, - kwargs={'config': config, - 'validate': validate}) + exchange = ExchangeResolver._load_exchange( + exchange_name, + kwargs={ + 'config': config, + 'validate': validate, + 'load_leverage_tiers': load_leverage_tiers} + ) except ImportError: logger.info( f"No {exchange_name} specific subclass found. Using the generic class instead.") diff --git a/freqtrade/rpc/api_server/deps.py b/freqtrade/rpc/api_server/deps.py index f5e61602e..81c013efa 100644 --- a/freqtrade/rpc/api_server/deps.py +++ b/freqtrade/rpc/api_server/deps.py @@ -37,7 +37,7 @@ def get_exchange(config=Depends(get_config)): if not ApiServer._exchange: from freqtrade.resolvers import ExchangeResolver ApiServer._exchange = ExchangeResolver.load_exchange( - config['exchange']['name'], config) + config['exchange']['name'], config, load_leverage_tiers=False) return ApiServer._exchange From 6e691a016d1ae3fcd18596d45ab48d24fecf4d35 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Jul 2022 10:24:59 +0200 Subject: [PATCH 29/30] Use leverage-tiers loading in tests --- tests/conftest.py | 2 +- tests/exchange/test_ccxt_compat.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3158e9ede..237c3fcc0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -148,7 +148,7 @@ def get_patched_exchange(mocker, config, api_mock=None, id='binance', patch_exchange(mocker, api_mock, id, mock_markets, mock_supported_modes) config['exchange']['name'] = id try: - exchange = ExchangeResolver.load_exchange(id, config) + exchange = ExchangeResolver.load_exchange(id, config, load_leverage_tiers=True) except ImportError: exchange = Exchange(config) return exchange diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 74106f28b..7bb52ccaf 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -137,7 +137,8 @@ def exchange_futures(request, exchange_conf, class_mocker): 'freqtrade.exchange.binance.Binance.fill_leverage_tiers') class_mocker.patch('freqtrade.exchange.exchange.Exchange.fetch_trading_fees') class_mocker.patch('freqtrade.exchange.okx.Okx.additional_exchange_init') - exchange = ExchangeResolver.load_exchange(request.param, exchange_conf, validate=True) + exchange = ExchangeResolver.load_exchange( + request.param, exchange_conf, validate=True, load_leverage_tiers=True) yield exchange, request.param From 83cac7bee2e6d94e47db336e1b83f2d55873c3d9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Jul 2022 10:51:13 +0200 Subject: [PATCH 30/30] Improve some more tests by adding proper orders --- tests/conftest.py | 19 ++++++++++++++++++- tests/rpc/test_rpc.py | 2 ++ tests/test_freqtradebot.py | 24 +++++++++++++++++++++++- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 237c3fcc0..ff3e1007f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2609,7 +2609,7 @@ def open_trade_usdt(): pair='ADA/USDT', open_rate=2.0, exchange='binance', - open_order_id='123456789', + open_order_id='123456789_exit', amount=30.0, fee_open=0.0, fee_close=0.0, @@ -2634,6 +2634,23 @@ def open_trade_usdt(): cost=trade.open_rate * trade.amount, order_date=trade.open_date, order_filled_date=trade.open_date, + ), + Order( + ft_order_side='exit', + ft_pair=trade.pair, + ft_is_open=True, + order_id='123456789_exit', + status="open", + symbol=trade.pair, + order_type="limit", + side="sell", + price=trade.open_rate, + average=trade.open_rate, + filled=trade.amount, + remaining=0, + cost=trade.open_rate * trade.amount, + order_date=trade.open_date, + order_filled_date=trade.open_date, ) ] return trade diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index d20646e60..6e19fcaf3 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -830,6 +830,8 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None: assert cancel_order_mock.call_count == 2 assert trade.amount == amount + trade = Trade.query.filter(Trade.id == '3').first() + # make an limit-sell open trade mocker.patch( 'freqtrade.exchange.Exchange.fetch_order', diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index a4b10fbcd..66cbd7d9b 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2759,6 +2759,8 @@ def test_check_handle_cancelled_exit( cancel_order_mock = MagicMock() limit_sell_order_old.update({"status": "canceled", 'filled': 0.0}) limit_sell_order_old['side'] = 'buy' if is_short else 'sell' + limit_sell_order_old['id'] = open_trade_usdt.open_order_id + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -3098,7 +3100,27 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: close_date=arrow.utcnow().datetime, exit_reason="sell_reason_whatever", ) - order = {'remaining': 1, + trade.orders = [ + Order( + ft_order_side='buy', + ft_pair=trade.pair, + ft_is_open=True, + order_id='123456', + status="closed", + symbol=trade.pair, + order_type="market", + side="buy", + price=trade.open_rate, + average=trade.open_rate, + filled=trade.amount, + remaining=0, + cost=trade.open_rate * trade.amount, + order_date=trade.open_date, + order_filled_date=trade.open_date, + ), + ] + order = {'id': "123456", + 'remaining': 1, 'amount': 1, 'status': "open"} reason = CANCEL_REASON['TIMEOUT']