From e8803477dfa269f9f95932f169c95554e22b504f Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Tue, 3 May 2022 22:56:55 -0600 Subject: [PATCH 01/37] exchange/exchange add param taker_or_maker to add_dry_order_fee --- freqtrade/exchange/exchange.py | 44 ++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 08bdab265..f0ff7e514 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -785,6 +785,26 @@ class Exchange: # Dry-run methods + def taker_or_maker( + self, + order_reason: Literal['entry', 'exit', 'stoploss'], # TODO: stoploss + ): + order_type = self._config['order_types'][order_reason] + if order_type == 'market' or order_reason == 'stoploss': + return 'taker' + else: + return ( + 'maker' if ( + ( + order_reason == 'entry' and + self._config['entry_pricing']['price_side'] == 'same' + ) or ( + order_reason == 'exit' and + self._config['exit_pricing']['price_side'] == 'same' + ) + ) else 'taker' + ) + def create_dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, leverage: float, params: Dict = {}, stop_loss: bool = False) -> Dict[str, Any]: @@ -824,7 +844,7 @@ class Exchange: 'filled': _amount, 'cost': (dry_order['amount'] * average) / leverage }) - dry_order = self.add_dry_order_fee(pair, dry_order) + dry_order = self.add_dry_order_fee(pair, dry_order, self.taker_or_maker('entry')) dry_order = self.check_dry_limit_order_filled(dry_order) @@ -832,12 +852,17 @@ class Exchange: # Copy order and close it - so the returned order is open unless it's a market order return dry_order - def add_dry_order_fee(self, pair: str, dry_order: Dict[str, Any]) -> Dict[str, Any]: + def add_dry_order_fee( + self, + pair: str, + dry_order: Dict[str, Any], + taker_or_maker: Literal['taker', 'maker'], + ) -> Dict[str, Any]: dry_order.update({ 'fee': { 'currency': self.get_pair_quote_currency(pair), - 'cost': dry_order['cost'] * self.get_fee(pair), - 'rate': self.get_fee(pair) + 'cost': dry_order['cost'] * self.get_fee(pair, taker_or_maker=taker_or_maker), + 'rate': self.get_fee(pair, taker_or_maker=taker_or_maker) } }) return dry_order @@ -917,7 +942,16 @@ class Exchange: 'filled': order['amount'], 'remaining': 0, }) - self.add_dry_order_fee(pair, order) + enter_long = not order['is_short'] and order['side'] == 'buy' + enter_short = order['is_short'] and order['side'] == 'sell' + entry_or_exit: Literal['entry', 'exit'] = ( + 'entry' if (enter_short or enter_long) else 'exit' + ) + self.add_dry_order_fee( + pair, + order, + self.taker_or_maker(entry_or_exit) + ) return order From 5d9aee6b7e6591ddb1953159223af8ebbb9cf6a1 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Tue, 3 May 2022 23:18:13 -0600 Subject: [PATCH 02/37] test_taker_or_maker --- tests/exchange/test_exchange.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 1368bcb85..3edb5187c 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -4957,3 +4957,24 @@ def test_stoploss_contract_size(mocker, default_conf, contract_size, order_amoun assert order['cost'] == 100 assert order['filled'] == 100 assert order['remaining'] == 100 + + +@pytest.mark.parametrize('order_reason,price_side,order_type,taker_or_maker', [ + ("entry", "same", "limit", "maker"), + ("exit", "same", "limit", "maker"), + ("entry", "other", "limit", "taker"), + ("exit", "other", "limit", "taker"), + ("entry", "same", "market", "maker"), + ("exit", "same", "market", "maker"), + ("entry", "other", "market", "taker"), + ("exit", "other", "market", "taker"), + ("stoploss", "same", "limit", "taker"), + ("stoploss", "same", "market", "taker"), + ("stoploss", "other", "limit", "taker"), + ("stoploss", "other", "market", "taker"), +]) +def test_taker_or_maker(mocker, default_conf, order_reason, price_side, order_type, taker_or_maker): + default_conf[f"{order_reason}_pricing"]["price_side"] = price_side + default_conf["order_types"][order_reason] = order_type + exchange = get_patched_exchange(mocker, default_conf) + assert exchange.taker_or_maker(order_reason) == taker_or_maker From dac9931b4a53aedeb4435483dcbb956203bb5530 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Tue, 3 May 2022 23:52:01 -0600 Subject: [PATCH 03/37] test_create_dry_run_order_fees --- tests/exchange/test_exchange.py | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 3edb5187c..ab51362a5 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1125,6 +1125,47 @@ def test_create_dry_run_order(default_conf, mocker, side, exchange_name, leverag assert order["cost"] == 1 * 200 / leverage +@pytest.mark.parametrize('side,is_short,order_reason', [ + ("buy", False, "entry"), + ("sell", False, "exit"), + ("buy", True, "exit"), + ("sell", True, "entry"), +]) +@pytest.mark.parametrize("order_type,price_side,fee", [ + ("limit", "same", 1.0), + ("limit", "other", 2.0), + ("market", "same", 2.0), + ("market", "other", 2.0), +]) +def test_create_dry_run_order_fees( + default_conf, + mocker, + side, + order_type, + is_short, + order_reason, + price_side, + fee, +): + default_conf[f"{order_reason}_pricing"]["price_side"] = "same" + default_conf["order_types"][order_reason] = order_type + mocker.patch( + 'freqtrade.exchange.Exchange.get_fee', + lambda symbol, taker_or_maker: 2.0 if taker_or_maker == 'taker' else 1.0 + ) + exchange = get_patched_exchange(mocker, default_conf) + + order = exchange.create_dry_run_order( + pair='ADA/USDT', + ordertype=order_type, + side=side, + amount=10, + rate=2.0, + ) + + assert order['ft_fee_base'] == fee + + @pytest.mark.parametrize("side,startprice,endprice", [ ("buy", 25.563, 25.566), ("sell", 25.566, 25.563) From 86ad5dd02a7ae680a1d566f7b83b55e5cf65f399 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 4 May 2022 00:08:41 -0600 Subject: [PATCH 04/37] test_exchange::test_taker_or_maker fixes --- tests/exchange/test_exchange.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index ab51362a5..a13dca4b6 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -5015,7 +5015,9 @@ def test_stoploss_contract_size(mocker, default_conf, contract_size, order_amoun ("stoploss", "other", "market", "taker"), ]) def test_taker_or_maker(mocker, default_conf, order_reason, price_side, order_type, taker_or_maker): - default_conf[f"{order_reason}_pricing"]["price_side"] = price_side + if order_reason != 'stoploss': + default_conf[f"{order_reason}_pricing"]["price_side"] = price_side + default_conf["order_types"] = {} default_conf["order_types"][order_reason] = order_type exchange = get_patched_exchange(mocker, default_conf) assert exchange.taker_or_maker(order_reason) == taker_or_maker From 10cbb5e67c32b184445f868366e4eef38afe5878 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 4 May 2022 00:10:09 -0600 Subject: [PATCH 05/37] test_exchange::test_taker_or_maker fixes --- tests/exchange/test_exchange.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index a13dca4b6..86928a2f6 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -5005,13 +5005,13 @@ def test_stoploss_contract_size(mocker, default_conf, contract_size, order_amoun ("exit", "same", "limit", "maker"), ("entry", "other", "limit", "taker"), ("exit", "other", "limit", "taker"), - ("entry", "same", "market", "maker"), - ("exit", "same", "market", "maker"), + ("stoploss", "same", "limit", "taker"), + ("stoploss", "other", "limit", "taker"), + ("entry", "same", "market", "taker"), + ("exit", "same", "market", "taker"), ("entry", "other", "market", "taker"), ("exit", "other", "market", "taker"), - ("stoploss", "same", "limit", "taker"), ("stoploss", "same", "market", "taker"), - ("stoploss", "other", "limit", "taker"), ("stoploss", "other", "market", "taker"), ]) def test_taker_or_maker(mocker, default_conf, order_reason, price_side, order_type, taker_or_maker): From 8b2535a8da3eabbbb53a70eeebb6348afaf0edd6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Jul 2022 15:42:17 +0200 Subject: [PATCH 06/37] Update Typing for fees --- freqtrade/constants.py | 1 + freqtrade/exchange/exchange.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index ce7c0ff83..6d74ceafd 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -542,3 +542,4 @@ TradeList = List[List] LongShort = Literal['long', 'short'] EntryExit = Literal['entry', 'exit'] BuySell = Literal['buy', 'sell'] +MakerTaker = Literal['maker', 'taker'] diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index db7a8ce41..2deb2b70d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -20,7 +20,7 @@ from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, Precise, decimal_to_ from pandas import DataFrame from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell, - EntryExit, ListPairsWithTimeframes, PairWithTimeframe) + EntryExit, ListPairsWithTimeframes, MakerTaker, PairWithTimeframe) from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, TradingMode from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError, @@ -870,7 +870,8 @@ class Exchange: 'filled': _amount, 'cost': (dry_order['amount'] * average) / leverage }) - dry_order = self.add_dry_order_fee(pair, dry_order, self.taker_or_maker('entry')) + # market orders will always incurr taker fees + dry_order = self.add_dry_order_fee(pair, dry_order, 'taker') dry_order = self.check_dry_limit_order_filled(dry_order) @@ -882,7 +883,7 @@ class Exchange: self, pair: str, dry_order: Dict[str, Any], - taker_or_maker: Literal['taker', 'maker'], + taker_or_maker: MakerTaker, ) -> Dict[str, Any]: dry_order.update({ 'fee': { @@ -970,7 +971,7 @@ class Exchange: }) enter_long = not order['is_short'] and order['side'] == 'buy' enter_short = order['is_short'] and order['side'] == 'sell' - entry_or_exit: Literal['entry', 'exit'] = ( + entry_or_exit: EntryExit = ( 'entry' if (enter_short or enter_long) else 'exit' ) self.add_dry_order_fee( @@ -1635,7 +1636,7 @@ class Exchange: @retrier def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, - price: float = 1, taker_or_maker: str = 'maker') -> float: + price: float = 1, taker_or_maker: MakerTaker = 'maker') -> float: try: if self._config['dry_run'] and self._config.get('fee', None) is not None: return self._config['fee'] From 4172f92bfc492a140103d79027fc326348183cc0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Jul 2022 17:03:45 +0200 Subject: [PATCH 07/37] simplify dry-run taker/maker selection --- freqtrade/exchange/exchange.py | 33 ++++--------------------- tests/exchange/test_exchange.py | 43 ++++++++++++--------------------- 2 files changed, 20 insertions(+), 56 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 2deb2b70d..efb33ee74 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -811,26 +811,6 @@ class Exchange: # Dry-run methods - def taker_or_maker( - self, - order_reason: Literal['entry', 'exit', 'stoploss'], # TODO: stoploss - ): - order_type = self._config['order_types'][order_reason] - if order_type == 'market' or order_reason == 'stoploss': - return 'taker' - else: - return ( - 'maker' if ( - ( - order_reason == 'entry' and - self._config['entry_pricing']['price_side'] == 'same' - ) or ( - order_reason == 'exit' and - self._config['exit_pricing']['price_side'] == 'same' - ) - ) else 'taker' - ) - def create_dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, leverage: float, params: Dict = {}, stop_loss: bool = False) -> Dict[str, Any]: @@ -873,7 +853,7 @@ class Exchange: # market orders will always incurr taker fees dry_order = self.add_dry_order_fee(pair, dry_order, 'taker') - dry_order = self.check_dry_limit_order_filled(dry_order) + dry_order = self.check_dry_limit_order_filled(dry_order, immediate=True) self._dry_run_open_orders[dry_order["id"]] = dry_order # Copy order and close it - so the returned order is open unless it's a market order @@ -955,7 +935,8 @@ class Exchange: pass return False - def check_dry_limit_order_filled(self, order: Dict[str, Any]) -> Dict[str, Any]: + def check_dry_limit_order_filled( + self, order: Dict[str, Any], immediate: bool = False) -> Dict[str, Any]: """ Check dry-run limit order fill and update fee (if it filled). """ @@ -969,15 +950,11 @@ class Exchange: 'filled': order['amount'], 'remaining': 0, }) - enter_long = not order['is_short'] and order['side'] == 'buy' - enter_short = order['is_short'] and order['side'] == 'sell' - entry_or_exit: EntryExit = ( - 'entry' if (enter_short or enter_long) else 'exit' - ) + self.add_dry_order_fee( pair, order, - self.taker_or_maker(entry_or_exit) + 'taker' if immediate else 'maker', ) return order diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 0c4df8f5a..ff8b4b40c 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1160,23 +1160,33 @@ def test_create_dry_run_order_fees( price_side, fee, ): - default_conf[f"{order_reason}_pricing"]["price_side"] = "same" - default_conf["order_types"][order_reason] = order_type mocker.patch( 'freqtrade.exchange.Exchange.get_fee', - lambda symbol, taker_or_maker: 2.0 if taker_or_maker == 'taker' else 1.0 + side_effect=lambda symbol, taker_or_maker: 2.0 if taker_or_maker == 'taker' else 1.0 ) + mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', + return_value=price_side == 'other') exchange = get_patched_exchange(mocker, default_conf) order = exchange.create_dry_run_order( - pair='ADA/USDT', + pair='LTC/USDT', ordertype=order_type, side=side, amount=10, rate=2.0, + leverage=1.0 ) + if price_side == 'other' or order_type == 'market': + assert order['fee']['rate'] == fee + return + else: + assert order['fee'] is None - assert order['ft_fee_base'] == fee + mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', + return_value=price_side != 'other') + + order1 = exchange.fetch_dry_run_order(order['id']) + assert order1['fee']['rate'] == fee @pytest.mark.parametrize("side,startprice,endprice", [ @@ -5101,26 +5111,3 @@ def test_stoploss_contract_size(mocker, default_conf, contract_size, order_amoun assert order['cost'] == 100 assert order['filled'] == 100 assert order['remaining'] == 100 - - -@pytest.mark.parametrize('order_reason,price_side,order_type,taker_or_maker', [ - ("entry", "same", "limit", "maker"), - ("exit", "same", "limit", "maker"), - ("entry", "other", "limit", "taker"), - ("exit", "other", "limit", "taker"), - ("stoploss", "same", "limit", "taker"), - ("stoploss", "other", "limit", "taker"), - ("entry", "same", "market", "taker"), - ("exit", "same", "market", "taker"), - ("entry", "other", "market", "taker"), - ("exit", "other", "market", "taker"), - ("stoploss", "same", "market", "taker"), - ("stoploss", "other", "market", "taker"), -]) -def test_taker_or_maker(mocker, default_conf, order_reason, price_side, order_type, taker_or_maker): - if order_reason != 'stoploss': - default_conf[f"{order_reason}_pricing"]["price_side"] = price_side - default_conf["order_types"] = {} - default_conf["order_types"][order_reason] = order_type - exchange = get_patched_exchange(mocker, default_conf) - assert exchange.taker_or_maker(order_reason) == taker_or_maker From 423af371c0ec8822f03cd9f10528e2c0d58e3512 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Jul 2022 17:59:05 +0200 Subject: [PATCH 08/37] Simplify calculation by calling "get_fee" only once --- freqtrade/exchange/exchange.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index efb33ee74..a430cdac5 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -865,11 +865,12 @@ class Exchange: dry_order: Dict[str, Any], taker_or_maker: MakerTaker, ) -> Dict[str, Any]: + fee = self.get_fee(pair, taker_or_maker=taker_or_maker) dry_order.update({ 'fee': { 'currency': self.get_pair_quote_currency(pair), - 'cost': dry_order['cost'] * self.get_fee(pair, taker_or_maker=taker_or_maker), - 'rate': self.get_fee(pair, taker_or_maker=taker_or_maker) + 'cost': dry_order['cost'] * fee, + 'rate': fee } }) return dry_order From 9347677c6061648517767dce5916dcf0394c71e6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Jul 2022 19:33:26 +0200 Subject: [PATCH 09/37] Uppdate pricecontours test to not recreate backtesting every loop in hopes to fix random failure --- tests/optimize/test_backtesting.py | 61 ++++++++++++++++++------------ 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 6912184aa..0b964c54a 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -90,28 +90,6 @@ def load_data_test(what, testdatadir): fill_missing=True)} -def simple_backtest(config, contour, mocker, testdatadir) -> None: - patch_exchange(mocker) - config['timeframe'] = '1m' - backtesting = Backtesting(config) - backtesting._set_strategy(backtesting.strategylist[0]) - - data = load_data_test(contour, testdatadir) - processed = backtesting.strategy.advise_all_indicators(data) - min_date, max_date = get_timerange(processed) - assert isinstance(processed, dict) - results = backtesting.backtest( - processed=processed, - start_date=min_date, - end_date=max_date, - max_open_trades=1, - position_stacking=False, - enable_protections=config.get('enable_protections', False), - ) - # results :: - return results - - # FIX: fixturize this? def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC'): data = history.load_data(datadir=datadir, timeframe='1m', pairs=[pair]) @@ -942,6 +920,7 @@ def test_backtest_dataprovider_analyzed_df(default_conf, fee, mocker, testdatadi def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatadir) -> None: # While this test IS a copy of test_backtest_pricecontours, it's needed to ensure # results do not carry-over to the next run, which is not given by using parametrize. + patch_exchange(mocker) default_conf['protections'] = [ { "method": "CooldownPeriod", @@ -949,6 +928,7 @@ def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatad }] default_conf['enable_protections'] = True + default_conf['timeframe'] = '1m' mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) mocker.patch("freqtrade.exchange.Exchange.get_max_pair_stake_amount", return_value=float('inf')) @@ -959,12 +939,27 @@ def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatad ['sine', 9], ['raise', 10], ] + backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) + # While entry-signals are unrealistic, running backtesting # over and over again should not cause different results for [contour, numres] in tests: # Debug output for random test failure print(f"{contour}, {numres}") - assert len(simple_backtest(default_conf, contour, mocker, testdatadir)['results']) == numres + data = load_data_test(contour, testdatadir) + processed = backtesting.strategy.advise_all_indicators(data) + min_date, max_date = get_timerange(processed) + assert isinstance(processed, dict) + results = backtesting.backtest( + processed=processed, + start_date=min_date, + end_date=max_date, + max_open_trades=1, + position_stacking=False, + enable_protections=default_conf.get('enable_protections', False), + ) + assert len(results['results']) == numres @pytest.mark.parametrize('protections,contour,expected', [ @@ -990,7 +985,25 @@ def test_backtest_pricecontours(default_conf, fee, mocker, testdatadir, mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) # While entry-signals are unrealistic, running backtesting # over and over again should not cause different results - assert len(simple_backtest(default_conf, contour, mocker, testdatadir)['results']) == expected + + patch_exchange(mocker) + default_conf['timeframe'] = '1m' + backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) + + data = load_data_test(contour, testdatadir) + processed = backtesting.strategy.advise_all_indicators(data) + min_date, max_date = get_timerange(processed) + assert isinstance(processed, dict) + results = backtesting.backtest( + processed=processed, + start_date=min_date, + end_date=max_date, + max_open_trades=1, + position_stacking=False, + enable_protections=default_conf.get('enable_protections', False), + ) + assert len(results['results']) == expected def test_backtest_clash_buy_sell(mocker, default_conf, testdatadir): From 05a5ae4fcfac1423a67b80411535a0b2ba07da27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Jul 2022 22:28:46 +0200 Subject: [PATCH 10/37] Update plotting to use entry/exit terminology --- freqtrade/plot/plotting.py | 22 +++++++++++----------- tests/test_plotting.py | 36 ++++++++++++++++++------------------ 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index a64281156..f8e95300a 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -255,18 +255,18 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: """ # Trades can be empty if trades is not None and len(trades) > 0: - # Create description for sell summarizing the trade + # Create description for exit summarizing the trade trades['desc'] = trades.apply( lambda row: f"{row['profit_ratio']:.2%}, " + (f"{row['enter_tag']}, " if row['enter_tag'] is not None else "") + f"{row['exit_reason']}, " + f"{row['trade_duration']} min", axis=1) - trade_buys = go.Scatter( + trade_entries = go.Scatter( x=trades["open_date"], y=trades["open_rate"], mode='markers', - name='Trade buy', + name='Trade entry', text=trades["desc"], marker=dict( symbol='circle-open', @@ -277,12 +277,12 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: ) ) - trade_sells = go.Scatter( + trade_exits = go.Scatter( x=trades.loc[trades['profit_ratio'] > 0, "close_date"], y=trades.loc[trades['profit_ratio'] > 0, "close_rate"], text=trades.loc[trades['profit_ratio'] > 0, "desc"], mode='markers', - name='Sell - Profit', + name='Exit - Profit', marker=dict( symbol='square-open', size=11, @@ -290,12 +290,12 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: color='green' ) ) - trade_sells_loss = go.Scatter( + trade_exits_loss = go.Scatter( x=trades.loc[trades['profit_ratio'] <= 0, "close_date"], y=trades.loc[trades['profit_ratio'] <= 0, "close_rate"], text=trades.loc[trades['profit_ratio'] <= 0, "desc"], mode='markers', - name='Sell - Loss', + name='Exit - Loss', marker=dict( symbol='square-open', size=11, @@ -303,9 +303,9 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: color='red' ) ) - fig.add_trace(trade_buys, 1, 1) - fig.add_trace(trade_sells, 1, 1) - fig.add_trace(trade_sells_loss, 1, 1) + fig.add_trace(trade_entries, 1, 1) + fig.add_trace(trade_exits, 1, 1) + fig.add_trace(trade_exits_loss, 1, 1) else: logger.warning("No trades found.") return fig @@ -444,7 +444,7 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra Generate the graph from the data generated by Backtesting or from DB Volume will always be ploted in row2, so Row 1 and 3 are to our disposal for custom indicators :param pair: Pair to Display on the graph - :param data: OHLCV DataFrame containing indicators and buy/sell signals + :param data: OHLCV DataFrame containing indicators and entry/exit signals :param trades: All trades created :param indicators1: List containing Main plot indicators :param indicators2: List containing Sub plot indicators diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 9ee7a75c6..52e96e477 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -72,7 +72,7 @@ def test_add_indicators(default_conf, testdatadir, caplog): strategy = StrategyResolver.load_strategy(default_conf) - # Generate buy/sell signals and indicators + # Generate entry/exit signals and indicators data = strategy.analyze_ticker(data, {'pair': pair}) fig = generate_empty_figure() @@ -113,7 +113,7 @@ def test_add_areas(default_conf, testdatadir, caplog): ind_plain = {"macd": {"fill_to": "macdhist"}} strategy = StrategyResolver.load_strategy(default_conf) - # Generate buy/sell signals and indicators + # Generate entry/exit signals and indicators data = strategy.analyze_ticker(data, {'pair': pair}) fig = generate_empty_figure() @@ -165,24 +165,24 @@ def test_plot_trades(testdatadir, caplog): fig = plot_trades(fig, trades) figure = fig1.layout.figure - # Check buys - color, should be in first graph, ... - trade_buy = find_trace_in_fig_data(figure.data, 'Trade buy') - assert isinstance(trade_buy, go.Scatter) - assert trade_buy.yaxis == 'y' - assert len(trades) == len(trade_buy.x) - assert trade_buy.marker.color == 'cyan' - assert trade_buy.marker.symbol == 'circle-open' - assert trade_buy.text[0] == '3.99%, buy_tag, roi, 15 min' + # Check entry - color, should be in first graph, ... + trade_entries = find_trace_in_fig_data(figure.data, 'Trade entry') + assert isinstance(trade_entries, go.Scatter) + assert trade_entries.yaxis == 'y' + assert len(trades) == len(trade_entries.x) + assert trade_entries.marker.color == 'cyan' + assert trade_entries.marker.symbol == 'circle-open' + assert trade_entries.text[0] == '3.99%, buy_tag, roi, 15 min' - trade_sell = find_trace_in_fig_data(figure.data, 'Sell - Profit') - assert isinstance(trade_sell, go.Scatter) - assert trade_sell.yaxis == 'y' - assert len(trades.loc[trades['profit_ratio'] > 0]) == len(trade_sell.x) - assert trade_sell.marker.color == 'green' - assert trade_sell.marker.symbol == 'square-open' - assert trade_sell.text[0] == '3.99%, buy_tag, roi, 15 min' + trade_exit = find_trace_in_fig_data(figure.data, 'Exit - Profit') + assert isinstance(trade_exit, go.Scatter) + assert trade_exit.yaxis == 'y' + assert len(trades.loc[trades['profit_ratio'] > 0]) == len(trade_exit.x) + assert trade_exit.marker.color == 'green' + assert trade_exit.marker.symbol == 'square-open' + assert trade_exit.text[0] == '3.99%, buy_tag, roi, 15 min' - trade_sell_loss = find_trace_in_fig_data(figure.data, 'Sell - Loss') + trade_sell_loss = find_trace_in_fig_data(figure.data, 'Exit - Loss') assert isinstance(trade_sell_loss, go.Scatter) assert trade_sell_loss.yaxis == 'y' assert len(trades.loc[trades['profit_ratio'] <= 0]) == len(trade_sell_loss.x) From 46be1b8778c9729f16651c2b8f00ac78dd98de98 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 17 Jul 2022 07:21:42 +0200 Subject: [PATCH 11/37] Version bump ccxt to 1.90.88 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2bb3b4b75..49121a9da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.23.1 pandas==1.4.3 pandas-ta==0.3.14b -ccxt==1.90.47 +ccxt==1.90.88 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.4 aiohttp==3.8.1 From 2c6fb617a6574ac9342fdbaa937938e581de525a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 03:01:23 +0000 Subject: [PATCH 12/37] Bump jsonschema from 4.6.2 to 4.7.2 Bumps [jsonschema](https://github.com/python-jsonschema/jsonschema) from 4.6.2 to 4.7.2. - [Release notes](https://github.com/python-jsonschema/jsonschema/releases) - [Changelog](https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-jsonschema/jsonschema/compare/v4.6.2...v4.7.2) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 49121a9da..78a327988 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ arrow==1.2.2 cachetools==4.2.2 requests==2.28.1 urllib3==1.26.10 -jsonschema==4.6.2 +jsonschema==4.7.2 TA-Lib==0.4.24 technical==1.3.0 tabulate==0.8.10 From 5f820ab0a64a0ebe0152c74901cb890d8a8a0cd9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 03:01:26 +0000 Subject: [PATCH 13/37] Bump pytest-asyncio from 0.18.3 to 0.19.0 Bumps [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) from 0.18.3 to 0.19.0. - [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases) - [Changelog](https://github.com/pytest-dev/pytest-asyncio/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.18.3...v0.19.0) --- updated-dependencies: - dependency-name: pytest-asyncio dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f2f77c2ba..ac7f4f619 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ flake8-tidy-imports==4.8.0 mypy==0.961 pre-commit==2.20.0 pytest==7.1.2 -pytest-asyncio==0.18.3 +pytest-asyncio==0.19.0 pytest-cov==3.0.0 pytest-mock==3.8.2 pytest-random-order==1.0.4 From cb63d5e3dfe9dc9b8275098ba8b7cf6dc446ac2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 03:01:31 +0000 Subject: [PATCH 14/37] Bump fastapi from 0.78.0 to 0.79.0 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.78.0 to 0.79.0. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.78.0...0.79.0) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 49121a9da..18c656bfb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ orjson==3.7.7 sdnotify==0.3.2 # API Server -fastapi==0.78.0 +fastapi==0.79.0 uvicorn==0.18.2 pyjwt==2.4.0 aiofiles==0.8.0 From f07ad7aa875a3c0c53de3c70b73330a90676d6c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 03:01:40 +0000 Subject: [PATCH 15/37] Bump ccxt from 1.90.88 to 1.90.89 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.90.88 to 1.90.89. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.90.88...1.90.89) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 49121a9da..9c9973ee8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.23.1 pandas==1.4.3 pandas-ta==0.3.14b -ccxt==1.90.88 +ccxt==1.90.89 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.4 aiohttp==3.8.1 From d2ef248781718ce27e3c59cad6539ba91adb01b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 03:01:43 +0000 Subject: [PATCH 16/37] Bump markdown from 3.3.7 to 3.4.1 Bumps [markdown](https://github.com/Python-Markdown/markdown) from 3.3.7 to 3.4.1. - [Release notes](https://github.com/Python-Markdown/markdown/releases) - [Commits](https://github.com/Python-Markdown/markdown/compare/3.3.7...3.4.1) --- updated-dependencies: - dependency-name: markdown dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 22d92c65d..a5fe8b921 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ -markdown==3.3.7 +markdown==3.4.1 mkdocs==1.3.0 mkdocs-material==8.3.9 mdx_truly_sane_lists==1.2 From ea523136fc29cdf62a14c59e85edfe089ac03b1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 03:01:49 +0000 Subject: [PATCH 17/37] Bump types-requests from 2.28.0 to 2.28.1 Bumps [types-requests](https://github.com/python/typeshed) from 2.28.0 to 2.28.1. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f2f77c2ba..435cb9b1c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -24,6 +24,6 @@ nbconvert==6.5.0 # mypy types types-cachetools==5.2.1 types-filelock==3.2.7 -types-requests==2.28.0 +types-requests==2.28.1 types-tabulate==0.8.11 types-python-dateutil==2.8.18 From 0daa9d3e57bfa09f98c17d14258dd9fad32803dd Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 18 Jul 2022 07:56:41 +0200 Subject: [PATCH 18/37] Bump types-requests in pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 59e7f6894..a23181c37 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: additional_dependencies: - types-cachetools==5.2.1 - types-filelock==3.2.7 - - types-requests==2.28.0 + - types-requests==2.28.1 - types-tabulate==0.8.11 - types-python-dateutil==2.8.18 # stages: [push] From 75e190ff1ddfecfd1796d99dbde4a060a18e96b7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 18 Jul 2022 20:15:07 +0200 Subject: [PATCH 19/37] Update sell-test without filled buy order --- tests/test_freqtradebot.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 4963e2b0a..e431e7ac3 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2060,8 +2060,9 @@ def test_update_trade_state_orderexception(mocker, default_conf_usdt, caplog) -> @pytest.mark.parametrize("is_short", [False, True]) def test_update_trade_state_sell( - default_conf_usdt, trades_for_order, limit_order_open, limit_order, is_short, mocker, + default_conf_usdt, trades_for_order, limit_order_open, limit_order, is_short, mocker ): + buy_order = limit_order[entry_side(is_short)] open_order = limit_order_open[exit_side(is_short)] l_order = limit_order[exit_side(is_short)] mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) @@ -2088,6 +2089,9 @@ def test_update_trade_state_sell( leverage=1, is_short=is_short, ) + order = Order.parse_from_ccxt_object(buy_order, 'LTC/ETH', entry_side(is_short)) + trade.orders.append(order) + order = Order.parse_from_ccxt_object(open_order, 'LTC/ETH', exit_side(is_short)) trade.orders.append(order) assert order.status == 'open' @@ -2787,6 +2791,7 @@ def test_manage_open_orders_partial( rpc_mock = patch_RPCManager(mocker) open_trade.is_short = is_short open_trade.leverage = leverage + open_trade.orders[0].ft_order_side = 'sell' if is_short else 'buy' limit_buy_order_old_partial['id'] = open_trade.open_order_id limit_buy_order_old_partial['side'] = 'sell' if is_short else 'buy' limit_buy_canceled = deepcopy(limit_buy_order_old_partial) @@ -2872,6 +2877,7 @@ def test_manage_open_orders_partial_except( limit_buy_order_old_partial_canceled, mocker ) -> None: open_trade.is_short = is_short + open_trade.orders[0].ft_order_side = 'sell' if is_short else 'buy' rpc_mock = patch_RPCManager(mocker) limit_buy_order_old_partial_canceled['id'] = open_trade.open_order_id limit_buy_order_old_partial['id'] = open_trade.open_order_id @@ -3626,7 +3632,7 @@ def test_execute_trade_exit_market_order( 'freqtrade.exchange.Exchange', fetch_ticker=ticker_usdt, get_fee=fee, - _is_dry_limit_order_filled=MagicMock(return_value=False), + _is_dry_limit_order_filled=MagicMock(return_value=True), ) patch_whitelist(mocker, default_conf_usdt) freqtrade = FreqtradeBot(default_conf_usdt) @@ -3642,7 +3648,8 @@ def test_execute_trade_exit_market_order( # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker_usdt_sell_up + fetch_ticker=ticker_usdt_sell_up, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) freqtrade.config['order_types']['exit'] = 'market' @@ -3655,7 +3662,7 @@ def test_execute_trade_exit_market_order( assert not trade.is_open assert trade.close_profit == profit_ratio - assert rpc_mock.call_count == 3 + assert rpc_mock.call_count == 4 last_msg = rpc_mock.call_args_list[-2][0][0] assert { 'type': RPCMessageType.EXIT, From b609dbcd8648937ed00785b0cab32faca1512859 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 19 Jul 2022 19:51:03 +0200 Subject: [PATCH 20/37] Update mdx_truly_sane_lists to be compatible with markdown again --- .gitignore | 2 ++ docs/requirements-docs.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 97f77f779..d6cec5225 100644 --- a/.gitignore +++ b/.gitignore @@ -80,6 +80,8 @@ instance/ # Sphinx documentation docs/_build/ +# Mkdocs documentation +site/ # PyBuilder target/ diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index a5fe8b921..a07f4f944 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.4.1 mkdocs==1.3.0 mkdocs-material==8.3.9 -mdx_truly_sane_lists==1.2 +mdx_truly_sane_lists==1.3 pymdown-extensions==9.5 jinja2==3.1.2 From 32c3f62934f97520490be1b009a5d0f5dc7dab4f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 22 Jul 2022 19:45:00 +0200 Subject: [PATCH 21/37] Fix documentation typo closes #7115 --- docs/stoploss.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index 6ddb485a4..249c40109 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -175,8 +175,8 @@ Before this, `stoploss` is used for the trailing stoploss. * assuming the asset now increases to 102$ * the stoploss will now be at 91.8$ - 10% below the highest observed rate * assuming the asset now increases to 103.5$ (above the offset configured) - * the stop loss will now be -2% of 103$ = 101.42$ - * now the asset drops in value to 102\$, the stop loss will still be 101.42$ and would trigger once price breaks below 101.42$ + * the stop loss will now be -2% of 103.5$ = 101.43$ + * now the asset drops in value to 102\$, the stop loss will still be 101.43$ and would trigger once price breaks below 101.43$ ### Trailing stop loss only once the trade has reached a certain offset From e97468964a46bc7464928a1626fa60f7b946ffbc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 08:42:20 +0200 Subject: [PATCH 22/37] Add support for --timeframe-detail in hyperopt fix #7070 --- docs/hyperopt.md | 23 ++++++++++++++++------- freqtrade/commands/arguments.py | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 55fe8f008..c9ec30056 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -40,13 +40,15 @@ pip install -r requirements-hyperopt.txt ``` usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] - [-i TIMEFRAME] [--timerange TIMERANGE] + [--recursive-strategy-search] [-i TIMEFRAME] + [--timerange TIMERANGE] [--data-format-ohlcv {json,jsongz,hdf5}] [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [-p PAIRS [PAIRS ...]] [--hyperopt-path PATH] [--eps] [--dmmp] [--enable-protections] - [--dry-run-wallet DRY_RUN_WALLET] [-e INT] + [--dry-run-wallet DRY_RUN_WALLET] + [--timeframe-detail TIMEFRAME_DETAIL] [-e INT] [--spaces {all,buy,sell,roi,stoploss,trailing,protection,default} [{all,buy,sell,roi,stoploss,trailing,protection,default} ...]] [--print-all] [--no-color] [--print-json] [-j JOBS] [--random-state INT] [--min-trades INT] @@ -89,6 +91,9 @@ optional arguments: --dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET Starting balance, used for backtesting / hyperopt and dry-runs. + --timeframe-detail TIMEFRAME_DETAIL + Specify detail timeframe for backtesting (`1m`, `5m`, + `30m`, `1h`, `1d`). -e INT, --epochs INT Specify number of epochs (default: 100). --spaces {all,buy,sell,roi,stoploss,trailing,protection,default} [{all,buy,sell,roi,stoploss,trailing,protection,default} ...] Specify which parameters to hyperopt. Space-separated @@ -146,7 +151,9 @@ Strategy arguments: Specify strategy class name which will be used by the bot. --strategy-path PATH Specify additional strategy lookup path. - + --recursive-strategy-search + Recursively search for a strategy in the strategies + folder. ``` ### Hyperopt checklist @@ -867,10 +874,12 @@ You can also enable position stacking in the configuration file by explicitly se As hyperopt consumes a lot of memory (the complete data needs to be in memory once per parallel backtesting process), it's likely that you run into "out of memory" errors. To combat these, you have multiple options: -* reduce the amount of pairs -* reduce the timerange used (`--timerange `) -* reduce the number of parallel processes (`-j `) -* Increase the memory of your machine +* Reduce the amount of pairs. +* Reduce the timerange used (`--timerange `). +* Avoid using `--timeframe-detail` (this loads a lot of additional data into memory). +* Reduce the number of parallel processes (`-j `). +* Increase the memory of your machine. + ## The objective has been evaluated at this point before. diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 1e3e2845a..fc3eda14d 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -28,7 +28,7 @@ ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_pos ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", "position_stacking", "use_max_market_positions", - "enable_protections", "dry_run_wallet", + "enable_protections", "dry_run_wallet", "timeframe_detail", "epochs", "spaces", "print_all", "print_colorized", "print_json", "hyperopt_jobs", "hyperopt_random_state", "hyperopt_min_trades", From 5c4f60f376452783674819a2b2e02e0e6ce8acad Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 09:11:22 +0200 Subject: [PATCH 23/37] Improve configuration table formatting and ordering --- docs/configuration.md | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0f3069478..412571674 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -116,6 +116,9 @@ This is similar to using multiple `--config` parameters, but simpler in usage as The table below will list all configuration parameters available. Freqtrade can also load many options via command line (CLI) arguments (check out the commands `--help` output for details). + +### Configuration option prevalence + The prevalence for all Options is as follows: - CLI arguments override any other option @@ -123,6 +126,8 @@ The prevalence for all Options is as follows: - Configuration files are used in sequence (the last file wins) and override Strategy configurations. - Strategy configurations are only used if they are not set via configuration or command-line arguments. These options are marked with [Strategy Override](#parameters-in-the-strategy) in the below table. +### Parameters table + Mandatory parameters are marked as **Required**, which means that they are required to be set in one of the possible ways. | Parameter | Description | @@ -135,7 +140,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `amend_last_stake_amount` | Use reduced last stake amount if necessary. [More information below](#configuring-amount-per-trade).
*Defaults to `false`.*
**Datatype:** Boolean | `last_stake_amount_min_ratio` | Defines minimum stake amount that has to be left and executed. Applies only to the last stake amount when it's amended to a reduced value (i.e. if `amend_last_stake_amount` is set to `true`). [More information below](#configuring-amount-per-trade).
*Defaults to `0.5`.*
**Datatype:** Float (as ratio) | `amount_reserve_percent` | Reserve some amount in min pair stake amount. The bot will reserve `amount_reserve_percent` + stoploss value when calculating min pair stake amount in order to avoid possible trade refusals.
*Defaults to `0.05` (5%).*
**Datatype:** Positive Float as ratio. -| `timeframe` | The timeframe to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** String +| `timeframe` | The timeframe to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). Usually missing in configuration, and specified in the strategy. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** String | `fiat_display_currency` | Fiat currency used to show your profits. [More information below](#what-values-can-be-used-for-fiat_display_currency).
**Datatype:** String | `dry_run` | **Required.** Define if the bot must be in Dry Run or production mode.
*Defaults to `true`.*
**Datatype:** Boolean | `dry_run_wallet` | Define the starting amount in stake currency for the simulated wallet used by the bot running in Dry Run mode.
*Defaults to `1000`.*
**Datatype:** Float @@ -148,13 +153,16 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md#trailing-stop-loss-only-once-the-trade-has-reached-a-certain-offset). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0` (no offset).*
**Datatype:** Float | `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `fee` | Fee used during backtesting / dry-runs. Should normally not be configured, which has freqtrade fall back to the exchange default fee. Set as ratio (e.g. 0.001 = 0.1%). Fee is applied twice for each trade, once when buying, once when selling.
**Datatype:** Float (as ratio) +| `futures_funding_rate` | User-specified funding rate to be used when historical funding rates are not available from the exchange. This does not overwrite real historical rates. It is recommended that this be set to 0 unless you are testing a specific coin and you understand how the funding rate will affect freqtrade's profit calculations. [More information here](leverage.md#unavailable-funding-rates)
*Defaults to None.*
**Datatype:** Float | `trading_mode` | Specifies if you want to trade regularly, trade with leverage, or trade contracts whose prices are derived from matching cryptocurrency prices. [leverage documentation](leverage.md).
*Defaults to `"spot"`.*
**Datatype:** String | `margin_mode` | When trading with leverage, this determines if the collateral owned by the trader will be shared or isolated to each trading pair [leverage documentation](leverage.md).
**Datatype:** String | `liquidation_buffer` | A ratio specifying how large of a safety net to place between the liquidation price and the stoploss to prevent a position from reaching the liquidation price [leverage documentation](leverage.md).
*Defaults to `0.05`.*
**Datatype:** Float +| | **Unfilled timeout** | `unfilledtimeout.entry` | **Required.** How long (in minutes or seconds) the bot will wait for an unfilled entry order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer | `unfilledtimeout.exit` | **Required.** How long (in minutes or seconds) the bot will wait for an unfilled exit order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer | `unfilledtimeout.unit` | Unit to use in unfilledtimeout setting. Note: If you set unfilledtimeout.unit to "seconds", "internals.process_throttle_secs" must be inferior or equal to timeout [Strategy Override](#parameters-in-the-strategy).
*Defaults to `minutes`.*
**Datatype:** String | `unfilledtimeout.exit_timeout_count` | How many times can exit orders time out. Once this number of timeouts is reached, an emergency exit is triggered. 0 to disable and allow unlimited order cancels. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0`.*
**Datatype:** Integer +| | **Pricing** | `entry_pricing.price_side` | Select the side of the spread the bot should look at to get the entry rate. [More information below](#buy-price-side).
*Defaults to `same`.*
**Datatype:** String (either `ask`, `bid`, `same` or `other`). | `entry_pricing.price_last_balance` | **Required.** Interpolate the bidding price. More information [below](#entry-price-without-orderbook-enabled). | `entry_pricing.use_order_book` | Enable entering using the rates in [Order Book Entry](#entry-price-with-orderbook-enabled).
*Defaults to `True`.*
**Datatype:** Boolean @@ -165,6 +173,8 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exit_pricing.price_last_balance` | Interpolate the exiting price. More information [below](#exit-price-without-orderbook-enabled). | `exit_pricing.use_order_book` | Enable exiting of open trades using [Order Book Exit](#exit-price-with-orderbook-enabled).
*Defaults to `True`.*
**Datatype:** Boolean | `exit_pricing.order_book_top` | Bot will use the top N rate in Order Book "price_side" to exit. I.e. a value of 2 will allow the bot to pick the 2nd ask rate in [Order Book Exit](#exit-price-with-orderbook-enabled)
*Defaults to `1`.*
**Datatype:** Positive Integer +| `custom_price_max_distance_ratio` | Configure maximum distance ratio between current and custom entry or exit price.
*Defaults to `0.02` 2%).*
**Datatype:** Positive float +| | **TODO** | `use_exit_signal` | Use exit signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
**Datatype:** Boolean | `exit_profit_only` | Wait until the bot reaches `exit_profit_offset` before taking an exit decision. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `exit_profit_offset` | Exit-signal is only active above this value. Only active in combination with `exit_profit_only=True`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0`.*
**Datatype:** Float (as ratio) @@ -172,8 +182,9 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `ignore_buying_expired_candle_after` | Specifies the number of seconds until a buy signal is no longer used.
**Datatype:** Integer | `order_types` | Configure order-types depending on the action (`"entry"`, `"exit"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict | `order_time_in_force` | Configure time in force for entry and exit orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict -| `custom_price_max_distance_ratio` | Configure maximum distance ratio between current and custom entry or exit price.
*Defaults to `0.02` 2%).*
**Datatype:** Positive float -| `recursive_strategy_search` | Set to `true` to recursively search sub-directories inside `user_data/strategies` for a strategy.
**Datatype:** Boolean +| `position_adjustment_enable` | Enables the strategy to use position adjustments (additional buys or sells). [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean +| `max_entry_position_adjustment` | Maximum additional order(s) for each open trade on top of the first entry Order. Set it to `-1` for unlimited additional orders. [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `-1`.*
**Datatype:** Positive Integer or -1 +| | **Exchange** | `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename).
**Datatype:** String | `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.
**Datatype:** Boolean | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String @@ -190,14 +201,19 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.skip_open_order_update` | Skips open order updates on startup should the exchange cause problems. Only relevant in live conditions.
*Defaults to `false`
**Datatype:** Boolean | `exchange.unknown_fee_rate` | Fallback value to use when calculating trading fees. This can be useful for exchanges which have fees in non-tradable currencies. The value provided here will be multiplied with the "fee cost".
*Defaults to `None`
**Datatype:** float | `exchange.log_responses` | Log relevant exchange responses. For debug mode only - use with care.
*Defaults to `false`
**Datatype:** Boolean -| `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
**Datatype:** Boolean +| | **Plugins** +| `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation of all possible configuration options. | `pairlists` | Define one or more pairlists to be used. [More information](plugins.md#pairlists-and-pairlist-handlers).
*Defaults to `StaticPairList`.*
**Datatype:** List of Dicts | `protections` | Define one or more protections to be used. [More information](plugins.md#protections).
**Datatype:** List of Dicts +| | **Telegram** | `telegram.enabled` | Enable the usage of Telegram.
**Datatype:** Boolean | `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `telegram.balance_dust_level` | Dust-level (in stake currency) - currencies with a balance below this will not be shown by `/balance`.
**Datatype:** float +| `telegram.reload` | Allow "reload" buttons on telegram messages.
*Defaults to `True`.
**Datatype:** boolean +| `telegram.notification_settings.*` | Detailed notification settings. Refer to the [telegram documentation](telegram-usage.md) for details.
**Datatype:** dictionary +| | **Webhook** | `webhook.enabled` | Enable usage of Webhook notifications
**Datatype:** Boolean | `webhook.url` | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String | `webhook.webhookentry` | Payload to send on entry. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String @@ -207,6 +223,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `webhook.webhookexitcancel` | Payload to send on exit order cancel. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String | `webhook.webhookexitfill` | Payload to send on exit order filled. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String | `webhook.webhookstatus` | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String +| | **Rest API / FreqUI** | `api_server.enabled` | Enable usage of API Server. See the [API Server documentation](rest-api.md) for more details.
**Datatype:** Boolean | `api_server.listen_ip_address` | Bind IP address. See the [API Server documentation](rest-api.md) for more details.
**Datatype:** IPv4 | `api_server.listen_port` | Bind Port. See the [API Server documentation](rest-api.md) for more details.
**Datatype:** Integer between 1024 and 65535 @@ -214,23 +231,22 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `api_server.username` | Username for API server. See the [API Server documentation](rest-api.md) for more details.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `api_server.password` | Password for API server. See the [API Server documentation](rest-api.md) for more details.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `bot_name` | Name of the bot. Passed via API to a client - can be shown to distinguish / name bots.
*Defaults to `freqtrade`*
**Datatype:** String -| `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite:///tradesv3.dryrun.sqlite` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
**Datatype:** String, SQLAlchemy connect string +| | **Other** | `initial_state` | Defines the initial application state. If set to stopped, then the bot has to be explicitly started via `/start` RPC command.
*Defaults to `stopped`.*
**Datatype:** Enum, either `stopped` or `running` | `force_entry_enable` | Enables the RPC Commands to force a Trade entry. More information below.
**Datatype:** Boolean | `disable_dataframe_checks` | Disable checking the OHLCV dataframe returned from the strategy methods for correctness. Only use when intentionally changing the dataframe and understand what you are doing. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `False`*.
**Datatype:** Boolean -| `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`.
**Datatype:** ClassName -| `strategy_path` | Adds an additional strategy lookup path (must be a directory).
**Datatype:** String | `internals.process_throttle_secs` | Set the process throttle, or minimum loop duration for one bot iteration loop. Value in second.
*Defaults to `5` seconds.*
**Datatype:** Positive Integer | `internals.heartbeat_interval` | Print heartbeat message every N seconds. Set to 0 to disable heartbeat messages.
*Defaults to `60` seconds.*
**Datatype:** Positive Integer or 0 | `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
**Datatype:** Boolean -| `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file.
**Datatype:** String +| `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`.
**Datatype:** ClassName +| `strategy_path` | Adds an additional strategy lookup path (must be a directory).
**Datatype:** String +| `recursive_strategy_search` | Set to `true` to recursively search sub-directories inside `user_data/strategies` for a strategy.
**Datatype:** Boolean | `user_data_dir` | Directory containing user data.
*Defaults to `./user_data/`*.
**Datatype:** String +| `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite:///tradesv3.dryrun.sqlite` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
**Datatype:** String, SQLAlchemy connect string +| `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file.
**Datatype:** String | `add_config_files` | Additional config files. These files will be loaded and merged with the current config file. The files are resolved relative to the initial file.
*Defaults to `[]`*.
**Datatype:** List of strings | `dataformat_ohlcv` | Data format to use to store historical candle (OHLCV) data.
*Defaults to `json`*.
**Datatype:** String | `dataformat_trades` | Data format to use to store historical trades data.
*Defaults to `jsongz`*.
**Datatype:** String -| `position_adjustment_enable` | Enables the strategy to use position adjustments (additional buys or sells). [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean -| `max_entry_position_adjustment` | Maximum additional order(s) for each open trade on top of the first entry Order. Set it to `-1` for unlimited additional orders. [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `-1`.*
**Datatype:** Positive Integer or -1 -| `futures_funding_rate` | User-specified funding rate to be used when historical funding rates are not available from the exchange. This does not overwrite real historical rates. It is recommended that this be set to 0 unless you are testing a specific coin and you understand how the funding rate will affect freqtrade's profit calculations. [More information here](leverage.md#unavailable-funding-rates)
*Defaults to None.*
**Datatype:** Float ### Parameters in the strategy From a02d02ac1251a1525545ee5fa3ba87fb61ac5899 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 14:43:52 +0200 Subject: [PATCH 24/37] Enhance protections tests to have orders in mock trade --- tests/plugins/test_protections.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 172e1f077..3c333200c 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -6,6 +6,7 @@ import pytest from freqtrade import constants from freqtrade.enums import ExitType from freqtrade.persistence import PairLocks, Trade +from freqtrade.persistence.trade_model import Order from freqtrade.plugins.protectionmanager import ProtectionManager from tests.conftest import get_patched_freqtradebot, log_has_re @@ -30,7 +31,37 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool, amount=0.01 / open_rate, exchange='binance', is_short=is_short, + leverage=1, ) + + trade.orders.append(Order( + ft_order_side=trade.entry_side, + order_id=f'{pair}-{trade.entry_side}-{trade.open_date}', + ft_pair=pair, + amount=trade.amount, + filled=trade.amount, + remaining=0, + price=open_rate, + average=open_rate, + status="closed", + order_type="market", + side=trade.entry_side, + )) + if not is_open: + trade.orders.append(Order( + ft_order_side=trade.exit_side, + order_id=f'{pair}-{trade.exit_side}-{trade.close_date}', + ft_pair=pair, + amount=trade.amount, + filled=trade.amount, + remaining=0, + price=open_rate * (2 - profit_rate if is_short else profit_rate), + average=open_rate * (2 - profit_rate if is_short else profit_rate), + status="closed", + order_type="market", + side=trade.exit_side, + )) + trade.recalc_open_trade_value() if not is_open: trade.close(open_rate * (2 - profit_rate if is_short else profit_rate)) From 80845807e113527fcbf2f716db62a4beb2ddae89 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 15:14:38 +0200 Subject: [PATCH 25/37] Improve some test resiliance --- tests/test_freqtradebot.py | 14 ++++++++------ tests/test_persistence.py | 8 ++++++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index e431e7ac3..a4b10fbcd 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2139,8 +2139,6 @@ def test_handle_trade( assert trade time.sleep(0.01) # Race condition fix - oobj = Order.parse_from_ccxt_object(enter_order, enter_order['symbol'], entry_side(is_short)) - trade.update_trade(oobj) assert trade.is_open is True freqtrade.wallets.update() @@ -2150,11 +2148,15 @@ def test_handle_trade( assert trade.open_order_id == exit_order['id'] # Simulate fulfilled LIMIT_SELL order for trade - oobj = Order.parse_from_ccxt_object(exit_order, exit_order['symbol'], exit_side(is_short)) - trade.update_trade(oobj) + trade.orders[-1].ft_is_open = False + trade.orders[-1].status = 'closed' + trade.orders[-1].filled = trade.orders[-1].remaining + trade.orders[-1].remaining = 0.0 - assert trade.close_rate == 2.0 if is_short else 2.2 - assert trade.close_profit == close_profit + trade.update_trade(trade.orders[-1]) + + assert trade.close_rate == (2.0 if is_short else 2.2) + assert pytest.approx(trade.close_profit) == close_profit assert trade.calc_profit(trade.close_rate) == 5.685 assert trade.close_date is not None assert trade.exit_reason == 'sell_signal1' diff --git a/tests/test_persistence.py b/tests/test_persistence.py index a09711048..529f6dc31 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -481,6 +481,7 @@ def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_ trade.open_order_id = 'something' oobj = Order.parse_from_ccxt_object(enter_order, 'ADA/USDT', entry_side) + trade.orders.append(oobj) trade.update_trade(oobj) assert trade.open_order_id is None assert trade.open_rate == open_rate @@ -496,11 +497,12 @@ def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_ trade.open_order_id = 'something' time_machine.move_to("2022-03-31 21:45:05 +00:00") oobj = Order.parse_from_ccxt_object(exit_order, 'ADA/USDT', exit_side) + trade.orders.append(oobj) trade.update_trade(oobj) assert trade.open_order_id is None assert trade.close_rate == close_rate - assert trade.close_profit == profit + assert pytest.approx(trade.close_profit) == profit assert trade.close_date is not None assert log_has_re(f"LIMIT_{exit_side.upper()} has been fulfilled for " r"Trade\(id=2, pair=ADA/USDT, amount=30.00000000, " @@ -529,6 +531,7 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, trade.open_order_id = 'something' oobj = Order.parse_from_ccxt_object(market_buy_order_usdt, 'ADA/USDT', 'buy') + trade.orders.append(oobj) trade.update_trade(oobj) assert trade.open_order_id is None assert trade.open_rate == 2.0 @@ -543,10 +546,11 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, trade.is_open = True trade.open_order_id = 'something' oobj = Order.parse_from_ccxt_object(market_sell_order_usdt, 'ADA/USDT', 'sell') + trade.orders.append(oobj) trade.update_trade(oobj) assert trade.open_order_id is None assert trade.close_rate == 2.2 - assert trade.close_profit == round(0.0945137157107232, 8) + assert pytest.approx(trade.close_profit) == 0.094513715710723 assert trade.close_date is not None assert log_has_re(r"MARKET_SELL has been fulfilled for Trade\(id=1, " r"pair=ADA/USDT, amount=30.00000000, is_short=False, leverage=1.0, " From 24a786bedd7e83a9e68e99041f6dd4d89fd018fc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 15:23:24 +0200 Subject: [PATCH 26/37] Update rpc test to contain sell order --- tests/rpc/test_rpc_telegram.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 91ee92fd7..f69b7e878 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -686,6 +686,7 @@ def test_profit_handle(default_conf_usdt, update, ticker_usdt, ticker_sell_up, f # Simulate fulfilled LIMIT_SELL order for trade oobj = Order.parse_from_ccxt_object( limit_sell_order_usdt, limit_sell_order_usdt['symbol'], 'sell') + trade.orders.append(oobj) trade.update_trade(oobj) trade.close_date = datetime.now(timezone.utc) @@ -707,7 +708,7 @@ def test_profit_handle(default_conf_usdt, update, ticker_usdt, ticker_sell_up, f assert '*Best Performing:* `ETH/USDT: 9.45%`' in msg_mock.call_args_list[-1][0][0] assert '*Max Drawdown:*' in msg_mock.call_args_list[-1][0][0] assert '*Profit factor:*' in msg_mock.call_args_list[-1][0][0] - assert '*Trading volume:* `60 USDT`' in msg_mock.call_args_list[-1][0][0] + assert '*Trading volume:* `126 USDT`' in msg_mock.call_args_list[-1][0][0] @pytest.mark.parametrize('is_short', [True, False]) From 7682c9ace70d6a6ad1e6a5f0c283d8dad067790c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 15:27:52 +0200 Subject: [PATCH 27/37] Update trade_close test to include orders --- tests/test_persistence.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 529f6dc31..838c4c22a 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -628,14 +628,41 @@ def test_trade_close(limit_buy_order_usdt, limit_sell_order_usdt, fee): open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=10), interest_rate=0.0005, exchange='binance', - trading_mode=margin + trading_mode=margin, + leverage=1.0, ) + trade.orders.append(Order( + ft_order_side=trade.entry_side, + order_id=f'{trade.pair}-{trade.entry_side}-{trade.open_date}', + ft_pair=trade.pair, + amount=trade.amount, + filled=trade.amount, + remaining=0, + price=trade.open_rate, + average=trade.open_rate, + status="closed", + order_type="limit", + side=trade.entry_side, + )) + trade.orders.append(Order( + ft_order_side=trade.exit_side, + order_id=f'{trade.pair}-{trade.exit_side}-{trade.open_date}', + ft_pair=trade.pair, + amount=trade.amount, + filled=trade.amount, + remaining=0, + price=2.2, + average=2.2, + status="closed", + order_type="limit", + side=trade.exit_side, + )) assert trade.close_profit is None assert trade.close_date is None assert trade.is_open is True trade.close(2.2) assert trade.is_open is False - assert trade.close_profit == round(0.0945137157107232, 8) + assert pytest.approx(trade.close_profit) == 0.094513715 assert trade.close_date is not None new_date = arrow.Arrow(2020, 2, 2, 15, 6, 1).datetime, From 2eb1d18c2a8f788fa47826ae8fca45795bcba56d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 19:56:38 +0200 Subject: [PATCH 28/37] Don't load leverage tiers when not necessary --- freqtrade/exchange/exchange.py | 5 +++-- freqtrade/freqtradebot.py | 3 ++- freqtrade/optimize/backtesting.py | 3 ++- freqtrade/resolvers/exchange_resolver.py | 13 +++++++++---- freqtrade/rpc/api_server/deps.py | 2 +- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index a430cdac5..11e37b953 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -88,7 +88,8 @@ class Exchange: # TradingMode.SPOT always supported and not required in this list ] - def __init__(self, config: Dict[str, Any], validate: bool = True) -> None: + def __init__(self, config: Dict[str, Any], validate: bool = True, + load_leverage_tiers: bool = False) -> None: """ Initializes this module with the given config, it does basic validation whether the specified exchange and pairs are valid. @@ -186,7 +187,7 @@ class Exchange: self.markets_refresh_interval: int = exchange_config.get( "markets_refresh_interval", 60) * 60 - if self.trading_mode != TradingMode.SPOT: + if self.trading_mode != TradingMode.SPOT and load_leverage_tiers: self.fill_leverage_tiers() self.additional_exchange_init() diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 2007f9b4e..43608cae7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -65,7 +65,8 @@ class FreqtradeBot(LoggingMixin): # Check config consistency here since strategies can set certain options validate_config_consistency(config) - self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) + self.exchange = ExchangeResolver.load_exchange( + self.config['exchange']['name'], self.config, load_leverage_tiers=True) init_db(self.config['db_url']) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index da28a8d93..4d16dc0f1 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -84,7 +84,8 @@ class Backtesting: self.processed_dfs: Dict[str, Dict] = {} self._exchange_name = self.config['exchange']['name'] - self.exchange = ExchangeResolver.load_exchange(self._exchange_name, self.config) + self.exchange = ExchangeResolver.load_exchange( + self._exchange_name, self.config, load_leverage_tiers=True) self.dataprovider = DataProvider(self.config, self.exchange) if self.config.get('strategy_list'): diff --git a/freqtrade/resolvers/exchange_resolver.py b/freqtrade/resolvers/exchange_resolver.py index 4dfbf445b..a2f572ff2 100644 --- a/freqtrade/resolvers/exchange_resolver.py +++ b/freqtrade/resolvers/exchange_resolver.py @@ -18,7 +18,8 @@ class ExchangeResolver(IResolver): object_type = Exchange @staticmethod - def load_exchange(exchange_name: str, config: dict, validate: bool = True) -> Exchange: + def load_exchange(exchange_name: str, config: dict, validate: bool = True, + load_leverage_tiers: bool = False) -> Exchange: """ Load the custom class from config parameter :param exchange_name: name of the Exchange to load @@ -29,9 +30,13 @@ class ExchangeResolver(IResolver): exchange_name = exchange_name.title() exchange = None try: - exchange = ExchangeResolver._load_exchange(exchange_name, - kwargs={'config': config, - 'validate': validate}) + exchange = ExchangeResolver._load_exchange( + exchange_name, + kwargs={ + 'config': config, + 'validate': validate, + 'load_leverage_tiers': load_leverage_tiers} + ) except ImportError: logger.info( f"No {exchange_name} specific subclass found. Using the generic class instead.") diff --git a/freqtrade/rpc/api_server/deps.py b/freqtrade/rpc/api_server/deps.py index f5e61602e..81c013efa 100644 --- a/freqtrade/rpc/api_server/deps.py +++ b/freqtrade/rpc/api_server/deps.py @@ -37,7 +37,7 @@ def get_exchange(config=Depends(get_config)): if not ApiServer._exchange: from freqtrade.resolvers import ExchangeResolver ApiServer._exchange = ExchangeResolver.load_exchange( - config['exchange']['name'], config) + config['exchange']['name'], config, load_leverage_tiers=False) return ApiServer._exchange From 6e691a016d1ae3fcd18596d45ab48d24fecf4d35 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Jul 2022 10:24:59 +0200 Subject: [PATCH 29/37] Use leverage-tiers loading in tests --- tests/conftest.py | 2 +- tests/exchange/test_ccxt_compat.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3158e9ede..237c3fcc0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -148,7 +148,7 @@ def get_patched_exchange(mocker, config, api_mock=None, id='binance', patch_exchange(mocker, api_mock, id, mock_markets, mock_supported_modes) config['exchange']['name'] = id try: - exchange = ExchangeResolver.load_exchange(id, config) + exchange = ExchangeResolver.load_exchange(id, config, load_leverage_tiers=True) except ImportError: exchange = Exchange(config) return exchange diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 74106f28b..7bb52ccaf 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -137,7 +137,8 @@ def exchange_futures(request, exchange_conf, class_mocker): 'freqtrade.exchange.binance.Binance.fill_leverage_tiers') class_mocker.patch('freqtrade.exchange.exchange.Exchange.fetch_trading_fees') class_mocker.patch('freqtrade.exchange.okx.Okx.additional_exchange_init') - exchange = ExchangeResolver.load_exchange(request.param, exchange_conf, validate=True) + exchange = ExchangeResolver.load_exchange( + request.param, exchange_conf, validate=True, load_leverage_tiers=True) yield exchange, request.param From 83cac7bee2e6d94e47db336e1b83f2d55873c3d9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Jul 2022 10:51:13 +0200 Subject: [PATCH 30/37] Improve some more tests by adding proper orders --- tests/conftest.py | 19 ++++++++++++++++++- tests/rpc/test_rpc.py | 2 ++ tests/test_freqtradebot.py | 24 +++++++++++++++++++++++- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 237c3fcc0..ff3e1007f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2609,7 +2609,7 @@ def open_trade_usdt(): pair='ADA/USDT', open_rate=2.0, exchange='binance', - open_order_id='123456789', + open_order_id='123456789_exit', amount=30.0, fee_open=0.0, fee_close=0.0, @@ -2634,6 +2634,23 @@ def open_trade_usdt(): cost=trade.open_rate * trade.amount, order_date=trade.open_date, order_filled_date=trade.open_date, + ), + Order( + ft_order_side='exit', + ft_pair=trade.pair, + ft_is_open=True, + order_id='123456789_exit', + status="open", + symbol=trade.pair, + order_type="limit", + side="sell", + price=trade.open_rate, + average=trade.open_rate, + filled=trade.amount, + remaining=0, + cost=trade.open_rate * trade.amount, + order_date=trade.open_date, + order_filled_date=trade.open_date, ) ] return trade diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index d20646e60..6e19fcaf3 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -830,6 +830,8 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None: assert cancel_order_mock.call_count == 2 assert trade.amount == amount + trade = Trade.query.filter(Trade.id == '3').first() + # make an limit-sell open trade mocker.patch( 'freqtrade.exchange.Exchange.fetch_order', diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index a4b10fbcd..66cbd7d9b 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2759,6 +2759,8 @@ def test_check_handle_cancelled_exit( cancel_order_mock = MagicMock() limit_sell_order_old.update({"status": "canceled", 'filled': 0.0}) limit_sell_order_old['side'] = 'buy' if is_short else 'sell' + limit_sell_order_old['id'] = open_trade_usdt.open_order_id + patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -3098,7 +3100,27 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: close_date=arrow.utcnow().datetime, exit_reason="sell_reason_whatever", ) - order = {'remaining': 1, + trade.orders = [ + Order( + ft_order_side='buy', + ft_pair=trade.pair, + ft_is_open=True, + order_id='123456', + status="closed", + symbol=trade.pair, + order_type="market", + side="buy", + price=trade.open_rate, + average=trade.open_rate, + filled=trade.amount, + remaining=0, + cost=trade.open_rate * trade.amount, + order_date=trade.open_date, + order_filled_date=trade.open_date, + ), + ] + order = {'id': "123456", + 'remaining': 1, 'amount': 1, 'status': "open"} reason = CANCEL_REASON['TIMEOUT'] From 70b7a254afdd701a9dd2a2d35c65668abac6088e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Jul 2022 16:51:48 +0200 Subject: [PATCH 31/37] Update some areas to use default docstring formatting --- docs/freqai.md | 11 ++- freqtrade/freqai/data_drawer.py | 22 +++--- freqtrade/freqai/freqai_interface.py | 75 +++++++++----------- freqtrade/strategy/interface.py | 11 ++- freqtrade/templates/FreqaiExampleStrategy.py | 11 ++- 5 files changed, 59 insertions(+), 71 deletions(-) diff --git a/docs/freqai.md b/docs/freqai.md index b2ee2407a..4060b5394 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -183,12 +183,11 @@ various configuration parameters which multiply the feature set such as `include (see convention below). I.e. user should not prepend any supporting metrics (e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the model. - :params: - :pair: pair to be used as informative - :df: strategy dataframe which will receive merges from informatives - :tf: timeframe of the dataframe which will modify the feature names - :informative: the dataframe associated with the informative pair - :coin: the name of the coin which will modify the feature names. + :param pair: pair to be used as informative + :param df: strategy dataframe which will receive merges from informatives + :param tf: timeframe of the dataframe which will modify the feature names + :param informative: the dataframe associated with the informative pair + :param coin: the name of the coin which will modify the feature names. """ with self.freqai.lock: diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index f9736a498..e18ecdbed 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -163,13 +163,12 @@ class FreqaiDataDrawer: Locate and load existing model metadata from persistent storage. If not located, create a new one and append the current pair to it and prepare it for its first training - :params: - metadata: dict = strategy furnished pair metadata - :returns: - model_filename: str = unique filename used for loading persistent objects from disk - trained_timestamp: int = the last time the coin was trained - coin_first: bool = If the coin is fresh without metadata - return_null_array: bool = Follower could not find pair metadata + :param pair: str: pair to lookup + :return: + model_filename: str = unique filename used for loading persistent objects from disk + trained_timestamp: int = the last time the coin was trained + coin_first: bool = If the coin is fresh without metadata + return_null_array: bool = Follower could not find pair metadata """ pair_in_dict = self.pair_dict.get(pair) data_path_set = self.pair_dict.get(pair, {}).get("data_path", None) @@ -277,13 +276,12 @@ class FreqaiDataDrawer: ) df = pd.concat([prepend_df, df], axis=0) - def attach_return_values_to_return_dataframe(self, pair: str, dataframe) -> DataFrame: + def attach_return_values_to_return_dataframe( + self, pair: str, dataframe: DataFrame) -> DataFrame: """ Attach the return values to the strat dataframe - :params: - dataframe: DataFrame = strat dataframe - :returns: - dataframe: DataFrame = strat dataframe with return values attached + :param dataframe: DataFrame = strategy dataframe + :return: DataFrame = strat dataframe with return values attached """ df = self.model_return_values[pair] to_keep = [col for col in dataframe.columns if not col.startswith("&")] diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index ac8cf6e60..b88923285 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -129,8 +129,7 @@ class IFreqaiModel(ABC): Function designed to constantly scan pairs for retraining on a separate thread (intracandle) to improve model youth. This function is agnostic to data preparation/collection/storage, it simply trains on what ever data is available in the self.dd. - :params: - strategy: IStrategy = The user defined strategy class + :param strategy: IStrategy = The user defined strategy class """ while 1: time.sleep(1) @@ -164,12 +163,11 @@ class IFreqaiModel(ABC): following the training window). FreqAI slides the window and sequentially builds the backtesting results before returning the concatenated results for the full backtesting period back to the strategy. - :params: - dataframe: DataFrame = strategy passed dataframe - metadata: Dict = pair metadata - dk: FreqaiDataKitchen = Data management/analysis tool assoicated to present pair only - :returns: - dk: FreqaiDataKitchen = Data management/analysis tool assoicated to present pair only + :param dataframe: DataFrame = strategy passed dataframe + :param metadata: Dict = pair metadata + :param dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only + :return: + FreqaiDataKitchen = Data management/analysis tool associated to present pair only """ self.pair_it += 1 @@ -239,13 +237,12 @@ class IFreqaiModel(ABC): """ The main broad execution for dry/live. This function will check if a retraining should be performed, and if so, retrain and reset the model. - :params: - dataframe: DataFrame = strategy passed dataframe - metadata: Dict = pair metadata - strategy: IStrategy = currently employed strategy - dk: FreqaiDataKitchen = Data management/analysis tool assoicated to present pair only + :param dataframe: DataFrame = strategy passed dataframe + :param metadata: Dict = pair metadata + :param strategy: IStrategy = currently employed strategy + dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only :returns: - dk: FreqaiDataKitchen = Data management/analysis tool assoicated to present pair only + dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only """ # update follower @@ -353,9 +350,9 @@ class IFreqaiModel(ABC): """ Ensure user is passing the proper feature set if they are reusing an `identifier` pointing to a folder holding existing models. - :params: - dataframe: DataFrame = strategy provided dataframe - dk: FreqaiDataKitchen = non-persistent data container/analyzer for current coin/bot loop + :param dataframe: DataFrame = strategy provided dataframe + :param dk: FreqaiDataKitchen = non-persistent data container/analyzer for + current coin/bot loop """ dk.find_features(dataframe) if "training_features_list_raw" in dk.data: @@ -461,13 +458,14 @@ class IFreqaiModel(ABC): """ Retreive data and train model in single threaded mode (only used if model directory is empty upon startup for dry/live ) - :params: - new_trained_timerange: TimeRange = the timerange to train the model on - metadata: dict = strategy provided metadata - strategy: IStrategy = user defined strategy object - dk: FreqaiDataKitchen = non-persistent data container for current coin/loop - data_load_timerange: TimeRange = the amount of data to be loaded for populate_any_indicators - (larger than new_trained_timerange so that new_trained_timerange does not contain any NaNs) + :param new_trained_timerange: TimeRange = the timerange to train the model on + :param metadata: dict = strategy provided metadata + :param strategy: IStrategy = user defined strategy object + :param dk: FreqaiDataKitchen = non-persistent data container for current coin/loop + :param data_load_timerange: TimeRange = the amount of data to be loaded + for populate_any_indicators + (larger than new_trained_timerange so that + new_trained_timerange does not contain any NaNs) """ corr_dataframes, base_dataframes = dk.get_base_and_corr_dataframes( @@ -515,11 +513,9 @@ class IFreqaiModel(ABC): """ Filter the training data and train a model to it. Train makes heavy use of the datahandler for storing, saving, loading, and analyzing the data. - :params: - :unfiltered_dataframe: Full dataframe for the current training period - :metadata: pair metadata from strategy. - :returns: - :model: Trained model which can be used to inference (self.predict) + :param unfiltered_dataframe: Full dataframe for the current training period + :param metadata: pair metadata from strategy. + :return: Trained model which can be used to inference (self.predict) """ @abstractmethod @@ -528,9 +524,8 @@ class IFreqaiModel(ABC): Most regressors use the same function names and arguments e.g. user can drop in LGBMRegressor in place of CatBoostRegressor and all data management will be properly handled by Freqai. - :params: - data_dictionary: Dict = the dictionary constructed by DataHandler to hold - all the training and test data/labels. + :param data_dictionary: Dict = the dictionary constructed by DataHandler to hold + all the training and test data/labels. """ return @@ -541,9 +536,9 @@ class IFreqaiModel(ABC): ) -> Tuple[DataFrame, npt.ArrayLike]: """ Filter the prediction features data and predict with it. - :param: - unfiltered_dataframe: Full dataframe for the current backtest period. - dk: FreqaiDataKitchen = Data management/analysis tool assoicated to present pair only + :param unfiltered_dataframe: Full dataframe for the current backtest period. + :param dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only + :param first: boolean = whether this is the first prediction or not. :return: :predictions: np.array of predictions :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove @@ -554,12 +549,10 @@ class IFreqaiModel(ABC): def return_values(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> DataFrame: """ User defines the dataframe to be returned to strategy here. - :params: - dataframe: DataFrame = the full dataframe for the current prediction (live) - or --timerange (backtesting) - dk: FreqaiDataKitchen = Data management/analysis tool assoicated to present pair only - :returns: - dataframe: DataFrame = dataframe filled with user defined data + :param dataframe: DataFrame = the full dataframe for the current prediction (live) + or --timerange (backtesting) + :param dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only + :return: dataframe: DataFrame = dataframe filled with user defined data """ return diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 83d16b6f6..431e67a98 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -566,12 +566,11 @@ class IStrategy(ABC, HyperStrategyMixin): additional features here, but must follow the naming convention. This method is *only* used in FreqaiDataKitchen class and therefore it is only called if FreqAI is active. - :params: - :pair: pair to be used as informative - :df: strategy dataframe which will receive merges from informatives - :tf: timeframe of the dataframe which will modify the feature names - :informative: the dataframe associated with the informative pair - :coin: the name of the coin which will modify the feature names. + :param pair: pair to be used as informative + :param df: strategy dataframe which will receive merges from informatives + :param tf: timeframe of the dataframe which will modify the feature names + :param informative: the dataframe associated with the informative pair + :param coin: the name of the coin which will modify the feature names. """ return df diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 7008008a3..58eb47532 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -74,12 +74,11 @@ class FreqaiExampleStrategy(IStrategy): (see convention below). I.e. user should not prepend any supporting metrics (e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the model. - :params: - :pair: pair to be used as informative - :df: strategy dataframe which will receive merges from informatives - :tf: timeframe of the dataframe which will modify the feature names - :informative: the dataframe associated with the informative pair - :coin: the name of the coin which will modify the feature names. + :param pair: pair to be used as informative + :param df: strategy dataframe which will receive merges from informatives + :param tf: timeframe of the dataframe which will modify the feature names + :param informative: the dataframe associated with the informative pair + :param coin: the name of the coin which will modify the feature names. """ with self.freqai.lock: From 1885deb632e52bcfb58c6e8bdb33178844477fd9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Jul 2022 16:54:39 +0200 Subject: [PATCH 32/37] More docstring changes --- freqtrade/freqai/freqai_interface.py | 8 ++++---- freqtrade/freqai/prediction_models/BaseRegressionModel.py | 7 +++---- freqtrade/freqai/prediction_models/BaseTensorFlowModel.py | 5 ++--- .../freqai/prediction_models/CatboostPredictionModel.py | 5 ++--- .../prediction_models/CatboostPredictionMultiModel.py | 5 ++--- .../freqai/prediction_models/LightGBMPredictionModel.py | 5 ++--- .../prediction_models/LightGBMPredictionMultiModel.py | 5 ++--- 7 files changed, 17 insertions(+), 23 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index b88923285..f5a1d667c 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -92,11 +92,11 @@ class IFreqaiModel(ABC): Entry point to the FreqaiModel from a specific pair, it will train a new model if necessary before making the prediction. - :params: - :dataframe: Full dataframe coming from strategy - it contains entire - backtesting timerange + additional historical data necessary to train + :param dataframe: Full dataframe coming from strategy - it contains entire + backtesting timerange + additional historical data necessary to train the model. - :metadata: pair metadata coming from strategy. + :param metadata: pair metadata coming from strategy. + :param strategy: Strategy to train on """ self.live = strategy.dp.runmode in (RunMode.DRY_RUN, RunMode.LIVE) diff --git a/freqtrade/freqai/prediction_models/BaseRegressionModel.py b/freqtrade/freqai/prediction_models/BaseRegressionModel.py index 2654b3726..c2fd53d0f 100644 --- a/freqtrade/freqai/prediction_models/BaseRegressionModel.py +++ b/freqtrade/freqai/prediction_models/BaseRegressionModel.py @@ -32,10 +32,9 @@ class BaseRegressionModel(IFreqaiModel): """ Filter the training data and train a model to it. Train makes heavy use of the datakitchen for storing, saving, loading, and analyzing the data. - :params: - :unfiltered_dataframe: Full dataframe for the current training period - :metadata: pair metadata from strategy. - :returns: + :param unfiltered_dataframe: Full dataframe for the current training period + :param metadata: pair metadata from strategy. + :return: :model: Trained model which can be used to inference (self.predict) """ diff --git a/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py b/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py index 098ff24dd..268bb00c9 100644 --- a/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py +++ b/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py @@ -31,9 +31,8 @@ class BaseTensorFlowModel(IFreqaiModel): """ Filter the training data and train a model to it. Train makes heavy use of the datakitchen for storing, saving, loading, and analyzing the data. - :params: - :unfiltered_dataframe: Full dataframe for the current training period - :metadata: pair metadata from strategy. + :param unfiltered_dataframe: Full dataframe for the current training period + :param metadata: pair metadata from strategy. :returns: :model: Trained model which can be used to inference (self.predict) """ diff --git a/freqtrade/freqai/prediction_models/CatboostPredictionModel.py b/freqtrade/freqai/prediction_models/CatboostPredictionModel.py index c69602025..f41760472 100644 --- a/freqtrade/freqai/prediction_models/CatboostPredictionModel.py +++ b/freqtrade/freqai/prediction_models/CatboostPredictionModel.py @@ -19,9 +19,8 @@ class CatboostPredictionModel(BaseRegressionModel): def fit(self, data_dictionary: Dict) -> Any: """ User sets up the training and test data to fit their desired model here - :params: - :data_dictionary: the dictionary constructed by DataHandler to hold - all the training and test data/labels. + :param data_dictionary: the dictionary constructed by DataHandler to hold + all the training and test data/labels. """ train_data = Pool( diff --git a/freqtrade/freqai/prediction_models/CatboostPredictionMultiModel.py b/freqtrade/freqai/prediction_models/CatboostPredictionMultiModel.py index 1b91fe0c6..17b5e6c68 100644 --- a/freqtrade/freqai/prediction_models/CatboostPredictionMultiModel.py +++ b/freqtrade/freqai/prediction_models/CatboostPredictionMultiModel.py @@ -20,9 +20,8 @@ class CatboostPredictionMultiModel(BaseRegressionModel): def fit(self, data_dictionary: Dict) -> Any: """ User sets up the training and test data to fit their desired model here - :params: - :data_dictionary: the dictionary constructed by DataHandler to hold - all the training and test data/labels. + :param data_dictionary: the dictionary constructed by DataHandler to hold + all the training and test data/labels. """ cbr = CatBoostRegressor( diff --git a/freqtrade/freqai/prediction_models/LightGBMPredictionModel.py b/freqtrade/freqai/prediction_models/LightGBMPredictionModel.py index 6a91837da..525566cf4 100644 --- a/freqtrade/freqai/prediction_models/LightGBMPredictionModel.py +++ b/freqtrade/freqai/prediction_models/LightGBMPredictionModel.py @@ -21,9 +21,8 @@ class LightGBMPredictionModel(BaseRegressionModel): Most regressors use the same function names and arguments e.g. user can drop in LGBMRegressor in place of CatBoostRegressor and all data management will be properly handled by Freqai. - :params: - :data_dictionary: the dictionary constructed by DataHandler to hold - all the training and test data/labels. + :param data_dictionary: the dictionary constructed by DataHandler to hold + all the training and test data/labels. """ eval_set = (data_dictionary["test_features"], data_dictionary["test_labels"]) diff --git a/freqtrade/freqai/prediction_models/LightGBMPredictionMultiModel.py b/freqtrade/freqai/prediction_models/LightGBMPredictionMultiModel.py index 89aad4323..4c51c9008 100644 --- a/freqtrade/freqai/prediction_models/LightGBMPredictionMultiModel.py +++ b/freqtrade/freqai/prediction_models/LightGBMPredictionMultiModel.py @@ -20,9 +20,8 @@ class LightGBMPredictionMultiModel(BaseRegressionModel): def fit(self, data_dictionary: Dict) -> Any: """ User sets up the training and test data to fit their desired model here - :params: - :data_dictionary: the dictionary constructed by DataHandler to hold - all the training and test data/labels. + :param data_dictionary: the dictionary constructed by DataHandler to hold + all the training and test data/labels. """ lgb = LGBMRegressor(**self.model_training_parameters) From 520ee3f7a1e8e741fd7ed23b08c5337520416351 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Jul 2022 17:07:45 +0200 Subject: [PATCH 33/37] Convert freqAI into packages --- freqtrade/freqai/__init__.py | 0 freqtrade/freqai/prediction_models/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 freqtrade/freqai/__init__.py create mode 100644 freqtrade/freqai/prediction_models/__init__.py diff --git a/freqtrade/freqai/__init__.py b/freqtrade/freqai/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/freqai/prediction_models/__init__.py b/freqtrade/freqai/prediction_models/__init__.py new file mode 100644 index 000000000..e69de29bb From ab587747fb4f686f687306be40ad69918df0913c Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Sun, 24 Jul 2022 23:32:02 +0200 Subject: [PATCH 34/37] first fix for follower path bug --- freqtrade/freqai/data_drawer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index e18ecdbed..98fca339d 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -151,7 +151,7 @@ class FreqaiDataDrawer: for pair in whitelist_pairs: self.follower_dict[pair] = {} - with open(self.follow_path, "w") as fp: + with open(self.follow_path_dict, "w") as fp: json.dump(self.follower_dict, fp, default=self.np_encoder) def np_encoder(self, object): From c9d46a5237a251193390a4c511737741773f851c Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Mon, 25 Jul 2022 09:24:40 +0200 Subject: [PATCH 35/37] finish bringing follow_mode up to date --- freqtrade/freqai/data_drawer.py | 2 +- freqtrade/freqai/freqai_interface.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 98fca339d..8aca4b371 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -151,7 +151,7 @@ class FreqaiDataDrawer: for pair in whitelist_pairs: self.follower_dict[pair] = {} - with open(self.follow_path_dict, "w") as fp: + with open(self.follower_dict_path, "w") as fp: json.dump(self.follower_dict, fp, default=self.np_encoder) def np_encoder(self, object): diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index f5a1d667c..fc352ea8b 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -287,7 +287,7 @@ class IFreqaiModel(ABC): elif self.follow_mode: dk.set_paths(metadata["pair"], trained_timestamp) logger.info( - "FreqAI instance set to follow_mode, finding existing pair" + "FreqAI instance set to follow_mode, finding existing pair " f"using { self.identifier }" ) From 4abc26b582445d4c407327b77be9b8920812dbc6 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Mon, 25 Jul 2022 10:48:04 +0200 Subject: [PATCH 36/37] add test for follow_mode --- tests/freqai/test_freqai_interface.py | 73 +++++++++++++++++++-------- 1 file changed, 53 insertions(+), 20 deletions(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 0bb2dac79..ce1a52bbf 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -34,26 +34,10 @@ def test_train_model_in_series_LightGBM(mocker, freqai_conf): freqai.train_model_in_series(new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) - assert ( - Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_model.joblib")) - .resolve() - .exists() - ) - assert ( - Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_metadata.json")) - .resolve() - .exists() - ) - assert ( - Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_trained_df.pkl")) - .resolve() - .exists() - ) - assert ( - Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_svm_model.joblib")) - .resolve() - .exists() - ) + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").is_file() shutil.rmtree(Path(freqai.dk.full_path)) @@ -161,3 +145,52 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): ) shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_follow_mode(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180110-20180130"}) + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dk.load_all_pair_histories(timerange) + + metadata = {"pair": "ADA/BTC"} + freqai.dd.set_pair_dict_info(metadata) + # freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.train_model_in_series(new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").is_file() + + # start the follower and ask it to predict on existing files + + freqai_conf.get("freqai", {}).update({"follow_mode": "true"}) + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd, freqai.live) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dk.load_all_pair_histories(timerange) + + df = strategy.dp.get_pair_dataframe('ADA/BTC', '5m') + freqai.start_live(df, metadata, strategy, freqai.dk) + + assert len(freqai.dk.return_dataframe.index) == 5702 + + shutil.rmtree(Path(freqai.dk.full_path)) From 7b105532d13bd0edab34f5a1913a9cc22d3d08c2 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Mon, 25 Jul 2022 11:46:59 +0200 Subject: [PATCH 37/37] fix mypy error and add test for principal component analysis --- freqtrade/freqai/freqai_interface.py | 5 ++-- .../prediction_models/BaseRegressionModel.py | 7 ++--- tests/freqai/test_freqai_interface.py | 27 +++++++++++++++++++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index fc352ea8b..c393420b5 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -65,7 +65,6 @@ class IFreqaiModel(ABC): self.data_split_parameters = config.get("freqai", {}).get("data_split_parameters") self.model_training_parameters = config.get("freqai", {}).get("model_training_parameters") self.feature_parameters = config.get("freqai", {}).get("feature_parameters") - self.model = None self.retrain = False self.first = True self.set_full_path() @@ -372,8 +371,8 @@ class IFreqaiModel(ABC): """ Base data cleaning method for train Any function inside this method should drop training data points from the filtered_dataframe - based on user decided logic. See FreqaiDataKitchen::remove_outliers() for an example - of how outlier data points are dropped from the dataframe used for training. + based on user decided logic. See FreqaiDataKitchen::use_SVM_to_remove_outliers() for an + example of how outlier data points are dropped from the dataframe used for training. """ if self.freqai_info.get("feature_parameters", {}).get( diff --git a/freqtrade/freqai/prediction_models/BaseRegressionModel.py b/freqtrade/freqai/prediction_models/BaseRegressionModel.py index c2fd53d0f..2db025fd6 100644 --- a/freqtrade/freqai/prediction_models/BaseRegressionModel.py +++ b/freqtrade/freqai/prediction_models/BaseRegressionModel.py @@ -1,6 +1,7 @@ import logging -from typing import Tuple +from typing import Any, Tuple +import numpy.typing as npt from pandas import DataFrame from freqtrade.freqai.data_kitchen import FreqaiDataKitchen @@ -28,7 +29,7 @@ class BaseRegressionModel(IFreqaiModel): def train( self, unfiltered_dataframe: DataFrame, pair: str, dk: FreqaiDataKitchen - ) -> Tuple[DataFrame, DataFrame]: + ) -> Any: """ Filter the training data and train a model to it. Train makes heavy use of the datakitchen for storing, saving, loading, and analyzing the data. @@ -83,7 +84,7 @@ class BaseRegressionModel(IFreqaiModel): def predict( self, unfiltered_dataframe: DataFrame, dk: FreqaiDataKitchen, first: bool = False - ) -> Tuple[DataFrame, DataFrame]: + ) -> Tuple[DataFrame, npt.ArrayLike]: """ Filter the prediction features data and predict with it. :param: unfiltered_dataframe: Full dataframe for the current backtest period. diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index ce1a52bbf..6699ef563 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -194,3 +194,30 @@ def test_follow_mode(mocker, freqai_conf): assert len(freqai.dk.return_dataframe.index) == 5702 shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_principal_component_analysis(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180110-20180130"}) + freqai_conf.get("freqai", {}).get("feature_parameters", {}).update( + {"princpial_component_analysis": "true"}) + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dk.load_all_pair_histories(timerange) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.train_model_in_series(new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_pca_object.pkl") + + shutil.rmtree(Path(freqai.dk.full_path))