From 88172fab824c50c9ccba0585137351e1ce031d73 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 Aug 2021 15:45:02 +0200 Subject: [PATCH 001/239] Allow "detailed" backtesting timeframe to look into the candle --- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 4 ++ freqtrade/configuration/configuration.py | 3 ++ freqtrade/optimize/backtesting.py | 54 ++++++++++++++++++++++-- 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 1143db394..a10ea5568 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -22,7 +22,7 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", "max_open_trades", "stake_amount", "fee", "pairs"] ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", - "enable_protections", "dry_run_wallet", + "enable_protections", "dry_run_wallet", "detail_timeframe", "strategy_list", "export", "exportfilename"] ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 215ed3f6e..a168a44cf 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -135,6 +135,10 @@ AVAILABLE_CLI_OPTIONS = { help='Override the value of the `stake_amount` configuration setting.', ), # Backtesting + "detail_timeframe": Arg( + '--timeframe-detail', + help='Specify detail timeframe for backtesting (`1m`, `5m`, `30m`, `1h`, `1d`).', + ), "position_stacking": Arg( '--eps', '--enable-position-stacking', help='Allow buying the same pair multiple times (position stacking).', diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 4dd5b7203..1d95dfb03 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -242,6 +242,9 @@ class Configuration: except ValueError: pass + self._args_to_config(config, argname='detail_timeframe', + logstring='Parameter --detail-timeframe detected, ' + 'using {} for intra-candle backtesting') self._args_to_config(config, argname='stake_amount', logstring='Parameter --stake-amount detected, ' 'overriding stake_amount to: {} ...') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index eecc7af54..2106f76c6 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -86,6 +86,16 @@ class Backtesting: "configuration or as cli argument `--timeframe 5m`") self.timeframe = str(self.config.get('timeframe')) self.timeframe_min = timeframe_to_minutes(self.timeframe) + # Load detail timeframe if specified + self.timeframe_detail = str(self.config.get('detail_timeframe', '')) + if self.timeframe_detail: + self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail) + if self.timeframe_min <= self.timeframe_detail_min: + raise OperationalException( + "Detail timeframe must be smaller than strategy timeframe.") + + else: + self.timeframe_detail_min = 0 self.pairlists = PairListManager(self.exchange, self.config) if 'VolumePairList' in self.pairlists.name_list: @@ -158,7 +168,7 @@ class Backtesting: conf['protections'] = strategy.protections self.protections = ProtectionManager(self.config, strategy.protections) - def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]: + def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange, Dict[str, DataFrame]]: """ Loads backtest data and returns the data combined with the timerange as tuple. @@ -174,6 +184,18 @@ class Backtesting: fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), ) + if self.timeframe_detail: + detail_data = history.load_data( + datadir=self.config['datadir'], + pairs=self.pairlists.whitelist, + timeframe=self.timeframe_detail, + timerange=self.timerange, + startup_candles=0, + fail_without_data=True, + data_format=self.config.get('dataformat_ohlcv', 'json'), + ) + else: + detail_data = None min_date, max_date = history.get_timerange(data) @@ -186,7 +208,7 @@ class Backtesting: self.required_startup, min_date) self.progress.set_new_value(1) - return data, self.timerange + return data, self.timerange, detail_data def prepare_backtest(self, enable_protections): """ @@ -318,7 +340,8 @@ class Backtesting: else: return sell_row[OPEN_IDX] - def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]: + def _get_sell_trade_entry_for_candle(self, trade: LocalTrade, + sell_row: Tuple) -> Optional[LocalTrade]: sell_candle_time = sell_row[DATE_IDX].to_pydatetime() sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], # type: ignore sell_candle_time, sell_row[BUY_IDX], @@ -346,6 +369,29 @@ class Backtesting: return None + def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]: + if self.timeframe_detail: + sell_candle_time = sell_row[DATE_IDX].to_pydatetime() + sell_candle_end = sell_candle_time + timedelta(minutes=self.timeframe_min) + + detail_data = self.detail_data[trade.pair] + detail_data = detail_data.loc[ + (detail_data['date'] >= sell_candle_time) & + (detail_data['date'] < sell_candle_end) + ] + detail_data['buy'] = sell_row[BUY_IDX] + detail_data['sell'] = sell_row[SELL_IDX] + headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high'] + for det_row in detail_data[headers].values.tolist(): + res = self._get_sell_trade_entry_for_candle(trade, det_row) + if res: + return res + + return None + + else: + return self._get_sell_trade_entry_for_candle(trade, sell_row) + def _enter_trade(self, pair: str, row: List) -> Optional[LocalTrade]: try: stake_amount = self.wallets.get_trade_stake_amount(pair, None) @@ -591,7 +637,7 @@ class Backtesting: """ data: Dict[str, Any] = {} - data, timerange = self.load_bt_data() + data, timerange, self.detail_data = self.load_bt_data() logger.info("Dataload complete. Calculating indicators") for strat in self.strategylist: From 8405ccc15eff2de0eb9183dd8d5243e60552559c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 Aug 2021 15:34:43 +0200 Subject: [PATCH 002/239] Seperate detail data loading from regular backest-data loading --- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 2 +- freqtrade/configuration/configuration.py | 4 +-- freqtrade/optimize/backtesting.py | 39 ++++++++++++++---------- freqtrade/optimize/optimize_reports.py | 1 + freqtrade/rpc/api_server/api_backtest.py | 3 ++ freqtrade/rpc/api_server/api_schemas.py | 1 + 7 files changed, 32 insertions(+), 20 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index a10ea5568..899998310 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -22,7 +22,7 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", "max_open_trades", "stake_amount", "fee", "pairs"] ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", - "enable_protections", "dry_run_wallet", "detail_timeframe", + "enable_protections", "dry_run_wallet", "timeframe_detail", "strategy_list", "export", "exportfilename"] ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index a168a44cf..8f10bbd0a 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -135,7 +135,7 @@ AVAILABLE_CLI_OPTIONS = { help='Override the value of the `stake_amount` configuration setting.', ), # Backtesting - "detail_timeframe": Arg( + "timeframe_detail": Arg( '--timeframe-detail', help='Specify detail timeframe for backtesting (`1m`, `5m`, `30m`, `1h`, `1d`).', ), diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 1d95dfb03..b3dfebe86 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -242,9 +242,9 @@ class Configuration: except ValueError: pass - self._args_to_config(config, argname='detail_timeframe', + self._args_to_config(config, argname='timeframe_detail', logstring='Parameter --detail-timeframe detected, ' - 'using {} for intra-candle backtesting') + 'using {} for intra-candle backtesting ...') self._args_to_config(config, argname='stake_amount', logstring='Parameter --stake-amount detected, ' 'overriding stake_amount to: {} ...') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2106f76c6..156ff48be 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -87,7 +87,7 @@ class Backtesting: self.timeframe = str(self.config.get('timeframe')) self.timeframe_min = timeframe_to_minutes(self.timeframe) # Load detail timeframe if specified - self.timeframe_detail = str(self.config.get('detail_timeframe', '')) + self.timeframe_detail = str(self.config.get('timeframe_detail', '')) if self.timeframe_detail: self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail) if self.timeframe_min <= self.timeframe_detail_min: @@ -96,6 +96,7 @@ class Backtesting: else: self.timeframe_detail_min = 0 + self.detail_data: Dict[str, DataFrame] = {} self.pairlists = PairListManager(self.exchange, self.config) if 'VolumePairList' in self.pairlists.name_list: @@ -168,7 +169,7 @@ class Backtesting: conf['protections'] = strategy.protections self.protections = ProtectionManager(self.config, strategy.protections) - def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange, Dict[str, DataFrame]]: + def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]: """ Loads backtest data and returns the data combined with the timerange as tuple. @@ -184,18 +185,6 @@ class Backtesting: fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), ) - if self.timeframe_detail: - detail_data = history.load_data( - datadir=self.config['datadir'], - pairs=self.pairlists.whitelist, - timeframe=self.timeframe_detail, - timerange=self.timerange, - startup_candles=0, - fail_without_data=True, - data_format=self.config.get('dataformat_ohlcv', 'json'), - ) - else: - detail_data = None min_date, max_date = history.get_timerange(data) @@ -208,7 +197,24 @@ class Backtesting: self.required_startup, min_date) self.progress.set_new_value(1) - return data, self.timerange, detail_data + return data, self.timerange + + def load_bt_data_detail(self) -> None: + """ + Loads backtest detail data (smaller timeframe) if necessary. + """ + if self.timeframe_detail: + self.detail_data = history.load_data( + datadir=self.config['datadir'], + pairs=self.pairlists.whitelist, + timeframe=self.timeframe_detail, + timerange=self.timerange, + startup_candles=0, + fail_without_data=True, + data_format=self.config.get('dataformat_ohlcv', 'json'), + ) + else: + self.detail_data = {} def prepare_backtest(self, enable_protections): """ @@ -637,7 +643,8 @@ class Backtesting: """ data: Dict[str, Any] = {} - data, timerange, self.detail_data = self.load_bt_data() + data, timerange = self.load_bt_data() + self.load_bt_data_detail() logger.info("Dataload complete. Calculating indicators") for strat in self.strategylist: diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 7bb60228a..8bde48670 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -368,6 +368,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame], 'max_open_trades_setting': (config['max_open_trades'] if config['max_open_trades'] != float('inf') else -1), 'timeframe': config['timeframe'], + 'timeframe_detail': config.get('timeframe_detail', ''), 'timerange': config.get('timerange', ''), 'enable_protections': config.get('enable_protections', False), 'strategy_name': strategy, diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 2fa66645b..4623c187e 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -46,11 +46,14 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac if ( not ApiServer._bt or lastconfig.get('timeframe') != strat.timeframe + or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail') or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0) or lastconfig.get('timerange') != btconfig['timerange'] ): from freqtrade.optimize.backtesting import Backtesting ApiServer._bt = Backtesting(btconfig) + if ApiServer._bt.timeframe_detail: + ApiServer._bt.load_bt_data_detail() # Only reload data if timeframe changed. if ( diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 318762136..3adbebc16 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -324,6 +324,7 @@ class PairHistory(BaseModel): class BacktestRequest(BaseModel): strategy: str timeframe: Optional[str] + timeframe_detail: Optional[str] timerange: Optional[str] max_open_trades: Optional[int] stake_amount: Optional[Union[float, str]] From 3406b889b62fbf39ee4dd3b334ae669e0ebd8d1b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 Aug 2021 16:56:49 +0200 Subject: [PATCH 003/239] First test --- freqtrade/configuration/configuration.py | 2 +- tests/optimize/test_backtesting.py | 108 +++++++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index b3dfebe86..94b108f2b 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -243,7 +243,7 @@ class Configuration: pass self._args_to_config(config, argname='timeframe_detail', - logstring='Parameter --detail-timeframe detected, ' + logstring='Parameter --timeframe-detail detected, ' 'using {} for intra-candle backtesting ...') self._args_to_config(config, argname='stake_amount', logstring='Parameter --stake-amount detected, ' diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 998b2d837..1b3fede72 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -441,6 +441,15 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> with pytest.raises(OperationalException, match='VolumePairList not allowed for backtesting.'): Backtesting(default_conf) + default_conf.update({ + 'pairlists': [{"method": "StaticPairList"}], + 'timeframe_detail': '1d', + }) + + with pytest.raises(OperationalException, + match='Detail timeframe must be smaller than strategy timeframe.'): + Backtesting(default_conf) + def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, tickers) -> None: mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) @@ -1042,3 +1051,102 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat assert 'LEFT OPEN TRADES REPORT' in captured.out assert '2017-11-14 21:17:00 -> 2017-11-14 22:58:00 | Max open trades : 1' in captured.out assert 'STRATEGY SUMMARY' in captured.out + + +@pytest.mark.filterwarnings("ignore:deprecated") +def test_backtest_start_multi_strat_nomock_detail(default_conf, mocker, + caplog, testdatadir, capsys): + # Tests detail-data loading + default_conf.update({ + "use_sell_signal": True, + "sell_profit_only": False, + "sell_profit_offset": 0.0, + "ignore_roi_if_buy_signal": False, + }) + patch_exchange(mocker) + result1 = pd.DataFrame({'pair': ['XRP/BTC', 'LTC/BTC'], + 'profit_ratio': [0.0, 0.0], + 'profit_abs': [0.0, 0.0], + 'open_date': pd.to_datetime(['2018-01-29 18:40:00', + '2018-01-30 03:30:00', ], utc=True + ), + 'close_date': pd.to_datetime(['2018-01-29 20:45:00', + '2018-01-30 05:35:00', ], utc=True), + 'trade_duration': [235, 40], + 'is_open': [False, False], + 'stake_amount': [0.01, 0.01], + 'open_rate': [0.104445, 0.10302485], + 'close_rate': [0.104969, 0.103541], + 'sell_reason': [SellType.ROI, SellType.ROI] + }) + result2 = pd.DataFrame({'pair': ['XRP/BTC', 'LTC/BTC', 'ETH/BTC'], + 'profit_ratio': [0.03, 0.01, 0.1], + 'profit_abs': [0.01, 0.02, 0.2], + 'open_date': pd.to_datetime(['2018-01-29 18:40:00', + '2018-01-30 03:30:00', + '2018-01-30 05:30:00'], utc=True + ), + 'close_date': pd.to_datetime(['2018-01-29 20:45:00', + '2018-01-30 05:35:00', + '2018-01-30 08:30:00'], utc=True), + 'trade_duration': [47, 40, 20], + 'is_open': [False, False, False], + 'stake_amount': [0.01, 0.01, 0.01], + 'open_rate': [0.104445, 0.10302485, 0.122541], + 'close_rate': [0.104969, 0.103541, 0.123541], + 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] + }) + backtestmock = MagicMock(side_effect=[ + { + 'results': result1, + 'config': default_conf, + 'locks': [], + 'rejected_signals': 20, + 'final_balance': 1000, + }, + { + 'results': result2, + 'config': default_conf, + 'locks': [], + 'rejected_signals': 20, + 'final_balance': 1000, + } + ]) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['XRP/ETH'])) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) + + patched_configuration_load_config_file(mocker, default_conf) + + args = [ + 'backtesting', + '--config', 'config.json', + '--datadir', str(testdatadir), + '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), + '--timeframe', '5m', + '--timeframe-detail', '1m', + '--strategy-list', + 'DefaultStrategy' + ] + args = get_args(args) + start_backtesting(args) + + # check the logs, that will contain the backtest result + exists = [ + 'Parameter -i/--timeframe detected ... Using timeframe: 5m ...', + 'Parameter --timeframe-detail detected, using 1m for intra-candle backtesting ...', + f'Using data directory: {testdatadir} ...', + 'Loading data from 2019-10-11 00:00:00 ' + 'up to 2019-10-13 11:10:00 (2 days).', + 'Backtesting with data from 2019-10-11 01:40:00 ' + 'up to 2019-10-13 11:10:00 (2 days).', + 'Running backtesting for Strategy DefaultStrategy', + ] + + for line in exists: + assert log_has(line, caplog) + + captured = capsys.readouterr() + assert 'BACKTESTING REPORT' in captured.out + assert 'SELL REASON STATS' in captured.out + assert 'LEFT OPEN TRADES REPORT' in captured.out From fa4ec9f83e337b4bd0bee1ca798a4e6c310758c9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 Aug 2021 17:36:02 +0200 Subject: [PATCH 004/239] Add explicit test for get_sell_trade_entry --- freqtrade/optimize/backtesting.py | 5 +- tests/optimize/test_backtesting.py | 88 +++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 156ff48be..4b52e104b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -376,7 +376,7 @@ class Backtesting: return None def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]: - if self.timeframe_detail: + if self.timeframe_detail and trade.pair in self.detail_data: sell_candle_time = sell_row[DATE_IDX].to_pydatetime() sell_candle_end = sell_candle_time + timedelta(minutes=self.timeframe_min) @@ -385,6 +385,9 @@ class Backtesting: (detail_data['date'] >= sell_candle_time) & (detail_data['date'] < sell_candle_end) ] + if len(detail_data) == 0: + # Fall back to "regular" data if no detail data was found for this candle + return self._get_sell_trade_entry_for_candle(trade, sell_row) detail_data['buy'] = sell_row[BUY_IDX] detail_data['sell'] = sell_row[SELL_IDX] headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high'] diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 1b3fede72..1b4285339 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1,7 +1,7 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument import random -from datetime import timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import MagicMock, PropertyMock @@ -500,7 +500,7 @@ def test_backtest__enter_trade(default_conf, fee, mocker) -> None: pair = 'UNITTEST/BTC' row = [ pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0), - 1, # Sell + 1, # Buy 0.001, # Open 0.0011, # Close 0, # Sell @@ -548,6 +548,88 @@ def test_backtest__enter_trade(default_conf, fee, mocker) -> None: backtesting.cleanup() +def test_backtest__get_sell_trade_entry(default_conf, fee, mocker) -> None: + default_conf['use_sell_signal'] = False + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) + patch_exchange(mocker) + default_conf['timeframe_detail'] = '1m' + default_conf['max_open_trades'] = 2 + backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) + pair = 'UNITTEST/BTC' + row = [ + pd.Timestamp(year=2020, month=1, day=1, hour=4, minute=55, tzinfo=timezone.utc), + 1, # Buy + 200, # Open + 201, # Close + 0, # Sell + 195, # Low + 201.5, # High + '', # Buy Signal Name + ] + + trade = backtesting._enter_trade(pair, row=row) + assert isinstance(trade, LocalTrade) + + row_sell = [ + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0, tzinfo=timezone.utc), + 0, # Buy + 200, # Open + 201, # Close + 0, # Sell + 195, # Low + 210.5, # High + '', # Buy Signal Name + ] + row_detail = pd.DataFrame( + [ + [ + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0, tzinfo=timezone.utc), + 1, 200, 199, 0, 197, 200.1, '', + ], [ + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=1, tzinfo=timezone.utc), + 0, 199, 199.5, 0, 199, 199.7, '', + ], [ + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=2, tzinfo=timezone.utc), + 0, 199.5, 200.5, 0, 199, 200.8, '', + ], [ + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=3, tzinfo=timezone.utc), + 0, 200.5, 210.5, 0, 193, 210.5, '', # ROI sell (?) + ], [ + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=4, tzinfo=timezone.utc), + 0, 200, 199, 0, 193, 200.1, '', + ], + ], columns=["date", "buy", "open", "close", "sell", "low", "high", "buy_tag"] + ) + + # No data available. + res = backtesting._get_sell_trade_entry(trade, row_sell) + assert res is not None + assert res.sell_reason == SellType.ROI.value + assert res.close_date_utc == datetime(2020, 1, 1, 5, 0, tzinfo=timezone.utc) + + # Enter new trade + trade = backtesting._enter_trade(pair, row=row) + assert isinstance(trade, LocalTrade) + # Assign empty ... no result. + backtesting.detail_data[pair] = pd.DataFrame( + [], columns=["date", "buy", "open", "close", "sell", "low", "high", "buy_tag"]) + + res = backtesting._get_sell_trade_entry(trade, row) + assert res is None + + # Assign backtest-detail data + backtesting.detail_data[pair] = row_detail + + res = backtesting._get_sell_trade_entry(trade, row_sell) + assert res is not None + assert res.sell_reason == SellType.ROI.value + # Sell at minute 3 (not available above!) + assert res.close_date_utc == datetime(2020, 1, 1, 5, 3, tzinfo=timezone.utc) + assert round(res.close_rate, 3) == round(209.0225, 3) + + def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: default_conf['use_sell_signal'] = False mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) @@ -1127,7 +1209,7 @@ def test_backtest_start_multi_strat_nomock_detail(default_conf, mocker, '--timeframe-detail', '1m', '--strategy-list', 'DefaultStrategy' - ] + ] args = get_args(args) start_backtesting(args) From 89581ad25cd7b81d418dc15d835c5b48591a3aba Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Aug 2021 08:24:31 +0200 Subject: [PATCH 005/239] Fix typo in protections hyperopt doc closes #5499 --- docs/hyperopt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 7377a63c0..1eb90f1bc 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -456,7 +456,7 @@ class MyAwesomeStrategy(IStrategy): "only_per_pair": False }) - return protection + return prot def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: # ... From 1895230afe642a12150857a254935632a2d9ee09 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Aug 2021 09:18:46 +0200 Subject: [PATCH 006/239] Clarify exception on load when markets could not be loaded closes #5498 --- freqtrade/exchange/exchange.py | 7 +++++++ tests/exchange/test_exchange.py | 11 +++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 26f034e41..ecf3302d8 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -352,9 +352,16 @@ class Exchange: def validate_stakecurrency(self, stake_currency: str) -> None: """ Checks stake-currency against available currencies on the exchange. + Only runs on startup. If markets have not been loaded, there's been a problem with + the connection to the exchange. :param stake_currency: Stake-currency to validate :raise: OperationalException if stake-currency is not available. """ + if not self._markets: + raise OperationalException( + 'Could not load markets, therefore cannot start. ' + 'Please investigate the above error for more details.' + ) quote_currencies = self.get_quote_currencies() if stake_currency not in quote_currencies: raise OperationalException( diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index f6ac4c459..42da5dddc 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -557,7 +557,7 @@ def test_reload_markets_exception(default_conf, mocker, caplog): @pytest.mark.parametrize("stake_currency", ['ETH', 'BTC', 'USDT']) -def test_validate_stake_currency(default_conf, stake_currency, mocker, caplog): +def test_validate_stakecurrency(default_conf, stake_currency, mocker, caplog): default_conf['stake_currency'] = stake_currency api_mock = MagicMock() type(api_mock).load_markets = MagicMock(return_value={ @@ -571,7 +571,7 @@ def test_validate_stake_currency(default_conf, stake_currency, mocker, caplog): Exchange(default_conf) -def test_validate_stake_currency_error(default_conf, mocker, caplog): +def test_validate_stakecurrency_error(default_conf, mocker, caplog): default_conf['stake_currency'] = 'XRP' api_mock = MagicMock() type(api_mock).load_markets = MagicMock(return_value={ @@ -587,6 +587,13 @@ def test_validate_stake_currency_error(default_conf, mocker, caplog): 'Available currencies are: BTC, ETH, USDT'): Exchange(default_conf) + type(api_mock).load_markets = MagicMock(side_effect=ccxt.NetworkError('No connection.')) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + + with pytest.raises(OperationalException, + match=r'Could not load markets, therefore cannot start\. Please.*'): + Exchange(default_conf) + def test_get_quote_currencies(default_conf, mocker): ex = get_patched_exchange(mocker, default_conf) From c14d8ea8277a4112bed80d072c9e3517c07f249c Mon Sep 17 00:00:00 2001 From: Pan Long Date: Sun, 29 Aug 2021 16:34:01 +0800 Subject: [PATCH 007/239] Export HDF5 and CBLOSC paths. This is needed if homebrew isn't installed in the standard path, say, /usr/local/. --- setup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.sh b/setup.sh index feb0241f8..e5f81578d 100755 --- a/setup.sh +++ b/setup.sh @@ -119,6 +119,7 @@ function install_mac_newer_python_dependencies() { echo "-------------------------" brew install hdf5 fi + export HDF5_DIR=$(brew --prefix) if [ ! $(brew --prefix --installed c-blosc 2>/dev/null) ] then @@ -127,6 +128,7 @@ function install_mac_newer_python_dependencies() { echo "-------------------------" brew install c-blosc fi + export CBLOSC_DIR=$(brew --prefix) } # Install bot MacOS From 20878290a0a34de24328c74dbed31323e2455ff0 Mon Sep 17 00:00:00 2001 From: Pan Long Date: Mon, 30 Aug 2021 01:02:48 +0800 Subject: [PATCH 008/239] Surround "unlimited" by double quotes in build config. --- freqtrade/commands/build_config_commands.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 852cab92e..7e3534921 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -66,16 +66,22 @@ def ask_user_config() -> Dict[str, Any]: { "type": "text", "name": "stake_amount", - "message": "Please insert your stake amount:", + "message": f"Please insert your stake amount (Integer or '{UNLIMITED_STAKE_AMOUNT}'):", "default": "0.01", "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val), + "filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"' + if val == UNLIMITED_STAKE_AMOUNT + else val }, { "type": "text", "name": "max_open_trades", "message": f"Please insert max_open_trades (Integer or '{UNLIMITED_STAKE_AMOUNT}'):", "default": "3", - "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_int(val) + "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_int(val), + "filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"' + if val == UNLIMITED_STAKE_AMOUNT + else val }, { "type": "text", From c17595b314fb1e6dcfff38695c51e5c07f2b38fe Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Sat, 28 Aug 2021 19:39:50 +0200 Subject: [PATCH 009/239] Docs: Mention Performance Warning for strategies Related to #5408 --- docs/strategy-customization.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index cfea60d22..194517b2b 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -781,6 +781,8 @@ Printing more than a few rows is also possible (simply use `print(dataframe)` i ## Common mistakes when developing strategies +### Peeking into the future while backtesting + Backtesting analyzes the whole time-range at once for performance reasons. Because of this, strategy authors need to make sure that strategies do not look-ahead into the future. This is a common pain-point, which can cause huge differences between backtesting and dry/live run methods, since they all use data which is not available during dry/live runs, so these strategies will perform well during backtesting, but will fail / perform badly in real conditions. @@ -791,6 +793,33 @@ The following lists some common patterns which should be avoided to prevent frus - don't use `dataframe['volume'].mean()`. This uses the full DataFrame for backtesting, including data from the future. Use `dataframe['volume'].rolling().mean()` instead - don't use `.resample('1h')`. This uses the left border of the interval, so moves data from an hour to the start of the hour. Use `.resample('1h', label='right')` instead. +### Performance warning + +When executing a strategy, one can sometimes be greeted by the following in the logs + +> PerformanceWarning: DataFrame is highly fragmented. + +This is a warning from [`pandas`](https://github.com/pandas-dev/pandas) and as the warning continues to say: + use `pd.concat(axis=1)`. For example + +```python +for i in range(100): + dataframe[i] = ta.indicator(dataframe, param=i) +``` + +should be rewritten to + +```python +frames = [dataframe] +for i in range(100): + frames.append({ + str(i): ta.indicator(dataframe, param=i) + }) + +# Append columns to existing dataframe +merged_frame = pd.concat(frames, axis=1) +``` + ## Further strategy ideas To get additional Ideas for strategies, head over to our [strategy repository](https://github.com/freqtrade/freqtrade-strategies). Feel free to use them as they are - but results will depend on the current market situation, pairs used etc. - therefore please backtest the strategy for your exchange/desired pairs first, evaluate carefully, use at your own risk. From c64ebeb6e28e274ced0d8fef9f2ad1bce3b7c56c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 03:01:05 +0000 Subject: [PATCH 010/239] Bump cryptography from 3.4.7 to 3.4.8 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.4.7 to 3.4.8. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.4.7...3.4.8) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1bccbd725..8305a83f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ pandas==1.3.2 ccxt==1.55.28 # Pin cryptography for now due to rust build errors with piwheels -cryptography==3.4.7 +cryptography==3.4.8 aiohttp==3.7.4.post0 SQLAlchemy==1.4.23 python-telegram-bot==13.7 From d0504c47ef5a67242947bd26e8ad80bc4dab6dc6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 03:01:09 +0000 Subject: [PATCH 011/239] Bump plotly from 5.2.1 to 5.3.0 Bumps [plotly](https://github.com/plotly/plotly.py) from 5.2.1 to 5.3.0. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v5.2.1...v5.3.0) --- updated-dependencies: - dependency-name: plotly dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index d835ed5d9..62836a729 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.2.1 +plotly==5.3.0 From 26451e8c0119f92c9f2db1e068ebfb5d64d83d51 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 03:01:14 +0000 Subject: [PATCH 012/239] Bump ccxt from 1.55.28 to 1.55.56 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.55.28 to 1.55.56. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.55.28...1.55.56) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1bccbd725..f6c1a984d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.21.2 pandas==1.3.2 -ccxt==1.55.28 +ccxt==1.55.56 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 From 9e0ab9c2ca03a0cced009c1c13ecbe922c1b15af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 03:01:19 +0000 Subject: [PATCH 013/239] Bump fastapi from 0.68.0 to 0.68.1 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.68.0 to 0.68.1. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.68.0...0.68.1) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1bccbd725..0d89322c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ python-rapidjson==1.4 sdnotify==0.3.2 # API Server -fastapi==0.68.0 +fastapi==0.68.1 uvicorn==0.15.0 pyjwt==2.1.0 aiofiles==0.7.0 From 45c6f906919a11029585c3810719c11e84c4cace Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 03:01:24 +0000 Subject: [PATCH 014/239] Bump mkdocs-material from 7.2.4 to 7.2.5 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 7.2.4 to 7.2.5. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/docs/changelog.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/7.2.4...7.2.5) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 8fa7341c9..d820c9412 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.2 -mkdocs-material==7.2.4 +mkdocs-material==7.2.5 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 From 2a0c95a2e7b2a75929cd1b017ce69d44805bab6c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 30 Aug 2021 20:00:52 +0200 Subject: [PATCH 015/239] Update freqtrade/commands/build_config_commands.py --- freqtrade/commands/build_config_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 7e3534921..1fe90e83a 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -66,7 +66,7 @@ def ask_user_config() -> Dict[str, Any]: { "type": "text", "name": "stake_amount", - "message": f"Please insert your stake amount (Integer or '{UNLIMITED_STAKE_AMOUNT}'):", + "message": f"Please insert your stake amount (Number or '{UNLIMITED_STAKE_AMOUNT}'):", "default": "0.01", "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val), "filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"' From 4cc1f2b4a4f5ce0f94fed3bef8374cf9eb4501ba Mon Sep 17 00:00:00 2001 From: Yehya <69044426+yehya-dev@users.noreply.github.com> Date: Tue, 31 Aug 2021 10:48:09 +0530 Subject: [PATCH 016/239] Update bot-basics.md Term usage mistake (The first in the pair is the base and second is the quote) --- docs/bot-basics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/bot-basics.md b/docs/bot-basics.md index e7ff27040..80443a0bf 100644 --- a/docs/bot-basics.md +++ b/docs/bot-basics.md @@ -7,7 +7,7 @@ This page provides you some basic concepts on how Freqtrade works and operates. * **Strategy**: Your trading strategy, telling the bot what to do. * **Trade**: Open position. * **Open Order**: Order which is currently placed on the exchange, and is not yet complete. -* **Pair**: Tradable pair, usually in the format of Quote/Base (e.g. XRP/USDT). +* **Pair**: Tradable pair, usually in the format of Base/Quote (e.g. XRP/USDT). * **Timeframe**: Candle length to use (e.g. `"5m"`, `"1h"`, ...). * **Indicators**: Technical indicators (SMA, EMA, RSI, ...). * **Limit order**: Limit orders which execute at the defined limit price or better. From da5f8c87aee1bef44ca905908ebd951d067e43c9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Aug 2021 07:18:32 +0200 Subject: [PATCH 017/239] Add stake_currency to strategy interface allows type-completion in editors --- freqtrade/strategy/interface.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index c51860011..91963f223 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -120,6 +120,8 @@ class IStrategy(ABC, HyperStrategyMixin): # and wallets - access to the current balance. dp: Optional[DataProvider] = None wallets: Optional[Wallets] = None + # Filled from configuration + stake_currency: str # container variable for strategy source code __source__: str = '' From 1cbe303434bed66ddf20496caa4cf3014a46d0b0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Aug 2021 19:58:08 +0200 Subject: [PATCH 018/239] Add documentation for --detail-timeframe --- docs/backtesting.md | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 89980c670..3e3bfc9fe 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -18,6 +18,7 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-p PAIRS [PAIRS ...]] [--eps] [--dmmp] [--enable-protections] [--dry-run-wallet DRY_RUN_WALLET] + [--timeframe-detail TIMEFRAME_DETAIL] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] [--export {none,trades}] [--export-filename PATH] @@ -55,6 +56,9 @@ optional arguments: --dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET Starting balance, used for backtesting / hyperopt and dry-runs. + --timeframe-detail TIMEFRAME_DETAIL + Specify detail timeframe for backtesting (`1m`, `5m`, + `30m`, `1h`, `1d`). --strategy-list STRATEGY_LIST [STRATEGY_LIST ...] Provide a space-separated list of strategies to backtest. Please note that ticker-interval needs to be @@ -425,7 +429,12 @@ It contains some useful key metrics about performance of your strategy on backte - `Drawdown Start` / `Drawdown End`: Start and end datetime for this largest drawdown (can also be visualized via the `plot-dataframe` sub-command). - `Market change`: Change of the market during the backtest period. Calculated as average of all pairs changes from the first to the last candle using the "close" column. -### Assumptions made by backtesting +### Further backtest-result analysis + +To further analyze your backtest results, you can [export the trades](#exporting-trades-to-file). +You can then load the trades to perform further analysis as shown in our [data analysis](data-analysis.md#backtesting) backtesting section. + +## Assumptions made by backtesting Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions: @@ -456,10 +465,30 @@ Also, keep in mind that past results don't guarantee future success. In addition to the above assumptions, strategy authors should carefully read the [Common Mistakes](strategy-customization.md#common-mistakes-when-developing-strategies) section, to avoid using data in backtesting which is not available in real market conditions. -### Further backtest-result analysis +### Improved backtest accuracy -To further analyze your backtest results, you can [export the trades](#exporting-trades-to-file). -You can then load the trades to perform further analysis as shown in our [data analysis](data-analysis.md#backtesting) backtesting section. +One big limitation of backtesting is it's inability to know how prices moved intra-candle (was high before close, or viceversa?). +So assuming you run backtesting with a 1h timeframe, there will be 4 prices for that candle (Open, High, Low, Close). + +While backtesting does take some assumptions (read above) about this - this can never be perfect, and will always be biased in one way or the other. +To mitigate this, freqtrade can use a lower (faster) timeframe to simulate intra-candle movements. + +To utilize this, you can append `--timeframe-detail 5m` to your regular backtesting command. + +``` bash +freqtrade backtesting --strategy AwesomeStrategy --timeframe 1h --timeframe-detail 5m +``` + +This will load 1h data as well as 5m data for the timeframe. The strategy will be analyzed with the 1h timeframe - and for every "open trade candle" (candles where a trade is open) the 5m data will be used to simulate intra-candle movements. +All callback functions (`custom_sell()`, `custom_stoploss()`, ... ) will be running for each 5m candle once the trade is opened (so 12 times in the above example of 1h timeframe, and 5m detailed timeframe). + +`--timeframe-detail` must be smaller than the original timeframe, otherwise backtesting will fail to start. + +Obviously this will require more memory (5m data is bigger than 1h data), and will also impact runtime (depending on the amount of trades and trade durations). +Also, data must be available / downloaded already. + +!!! Tip + You can use this function as the last part of strategy development, to ensure your strategy is not exploiting one of the [backtesting assumptions](#assumptions-made-by-backtesting). Strategies that perform similarly well with this mode have a good chance to perform well in dry/live modes too (although only forward-testing (dry-mode) can really confirm a strategy). ## Backtesting multiple strategies From 87fa49d52981d214d2e9854f0c9e097dd5ca6920 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Sep 2021 03:01:49 +0000 Subject: [PATCH 019/239] Bump python from 3.9.6-slim-buster to 3.9.7-slim-buster Bumps python from 3.9.6-slim-buster to 3.9.7-slim-buster. --- updated-dependencies: - dependency-name: python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 804b1086b..4c4722452 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.6-slim-buster as base +FROM python:3.9.7-slim-buster as base # Setup env ENV LANG C.UTF-8 From 93c1dff71bbae773b922794f956013e0d0b9c908 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Sep 2021 20:43:07 +0200 Subject: [PATCH 020/239] Allow adding new additional headers --- freqtrade/exchange/exchange.py | 9 ++++++++- tests/exchange/test_exchange.py | 7 +++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ecf3302d8..3dc295563 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -54,6 +54,9 @@ class Exchange: # Parameters to add directly to buy/sell calls (like agreeing to trading agreement) _params: Dict = {} + # Additional headers - added to the ccxt object + _headers: Dict = {} + # Dict to specify which options each exchange implements # This defines defaults, which can be selectively overridden by subclasses using _ft_has # or by specifying them in the configuration. @@ -169,7 +172,7 @@ class Exchange: asyncio.get_event_loop().run_until_complete(self._api_async.close()) def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt, - ccxt_kwargs: dict = None) -> ccxt.Exchange: + ccxt_kwargs: Dict = {}) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid ccxt instance. @@ -188,6 +191,10 @@ class Exchange: } if ccxt_kwargs: logger.info('Applying additional ccxt config: %s', ccxt_kwargs) + if self._headers: + # Inject static headers after the above output to not confuse users. + ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs) + if ccxt_kwargs: ex_config.update(ccxt_kwargs) try: diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 42da5dddc..144063c07 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -108,6 +108,13 @@ def test_init_ccxt_kwargs(default_conf, mocker, caplog): assert hasattr(ex._api_async, 'TestKWARG') assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog) assert log_has(asynclogmsg, caplog) + # Test additional headers case + Exchange._headers = {'hello': 'world'} + ex = Exchange(conf) + + assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog) + assert ex._api.headers == {'hello': 'world'} + Exchange._headers = {} def test_destroy(default_conf, mocker, caplog): From 19ad1654836992365af7536189095662205d768a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Sep 2021 08:48:53 +0200 Subject: [PATCH 021/239] Add time_in_force for kucoin --- docs/configuration.md | 4 ++-- docs/exchanges.md | 5 +++++ freqtrade/exchange/binance.py | 1 + freqtrade/exchange/exchange.py | 4 +++- freqtrade/exchange/kucoin.py | 2 ++ 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 09198e019..6ccea4c73 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -444,8 +444,8 @@ The possible values are: `gtc` (default), `fok` or `ioc`. ``` !!! Warning - This is ongoing work. For now, it is supported only for binance. - Please don't change the default value unless you know what you are doing and have researched the impact of using different values. + This is ongoing work. For now, it is supported only for binance and kucoin. + Please don't change the default value unless you know what you are doing and have researched the impact of using different values for your particular exchange. ### Exchange configuration diff --git a/docs/exchanges.md b/docs/exchanges.md index 5f54a524e..81ad670a3 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -4,6 +4,9 @@ This page combines common gotchas and informations which are exchange-specific a ## Binance +Binance supports [time_in_force](configuration.md#understand-order_time_in_force). + + !!! Tip "Stoploss on Exchange" Binance supports `stoploss_on_exchange` and uses stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it. @@ -115,6 +118,8 @@ Kucoin requires a passphrase for each api key, you will therefore need to add th "password": "your_exchange_api_key_password", ``` +Kucoin supports [time_in_force](configuration.md#understand-order_time_in_force). + ### Kucoin Blacklists For Kucoin, please add `"KCS/"` to your blacklist to avoid issues. diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 0c470cb24..189f5f481 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -18,6 +18,7 @@ class Binance(Exchange): _ft_has: Dict = { "stoploss_on_exchange": True, "order_time_in_force": ['gtc', 'fok', 'ioc'], + "time_in_force_parameter": "timeInForce", "ohlcv_candle_limit": 1000, "trades_pagination": "id", "trades_pagination_arg": "fromId", diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 3dc295563..80f20b17e 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -63,6 +63,7 @@ class Exchange: _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], + "time_in_force_parameter": "timeInForce", "ohlcv_params": {}, "ohlcv_candle_limit": 500, "ohlcv_partial_candle": True, @@ -723,7 +724,8 @@ class Exchange: params = self._params.copy() if time_in_force != 'gtc' and ordertype != 'market': - params.update({'timeInForce': time_in_force}) + param = self._ft_has.get('time_in_force_parameter', '') + params.update({param: time_in_force}) try: # Set the precision for amount and price(rate) as accepted by the exchange diff --git a/freqtrade/exchange/kucoin.py b/freqtrade/exchange/kucoin.py index 22886a1d8..5d818f6a2 100644 --- a/freqtrade/exchange/kucoin.py +++ b/freqtrade/exchange/kucoin.py @@ -21,4 +21,6 @@ class Kucoin(Exchange): _ft_has: Dict = { "l2_limit_range": [20, 100], "l2_limit_range_required": False, + "order_time_in_force": ['gtc', 'fok', 'ioc'], + "time_in_force_parameter": "timeInForce", } From e64ccd8fc111e6d5eeeb193509317f0be00f1d80 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Sep 2021 08:54:15 +0200 Subject: [PATCH 022/239] Add new_config section for kucoin --- freqtrade/commands/build_config_commands.py | 8 ++++++++ .../templates/subtemplates/exchange_kucoin.j2 | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 freqtrade/templates/subtemplates/exchange_kucoin.j2 diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 1fe90e83a..780ad4161 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -105,6 +105,8 @@ def ask_user_config() -> Dict[str, Any]: "bittrex", "kraken", "ftx", + "kucoin", + "gateio", Separator(), "other", ], @@ -128,6 +130,12 @@ def ask_user_config() -> Dict[str, Any]: "message": "Insert Exchange Secret", "when": lambda x: not x['dry_run'] }, + { + "type": "password", + "name": "exchange_key_password", + "message": "Insert Exchange API Key password", + "when": lambda x: not x['dry_run'] and x['exchange_name'] == 'kucoin' + }, { "type": "confirm", "name": "telegram", diff --git a/freqtrade/templates/subtemplates/exchange_kucoin.j2 b/freqtrade/templates/subtemplates/exchange_kucoin.j2 new file mode 100644 index 000000000..f9dfff663 --- /dev/null +++ b/freqtrade/templates/subtemplates/exchange_kucoin.j2 @@ -0,0 +1,18 @@ +"exchange": { + "name": "{{ exchange_name | lower }}", + "key": "{{ exchange_key }}", + "secret": "{{ exchange_secret }}", + "password": "{{ exchange_key_password }}", + "ccxt_config": { + "enableRateLimit": true + "rateLimit": 200 + }, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 200 + }, + "pair_whitelist": [ + ], + "pair_blacklist": [ + ] +} From 68f13173bcaf4d88c2835abc66f9d0b92bf7ce7e Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Sep 2021 08:59:08 +0200 Subject: [PATCH 023/239] Update new-config templates to use USDT by default --- freqtrade/commands/build_config_commands.py | 4 ++-- .../subtemplates/exchange_binance.j2 | 15 ------------- .../subtemplates/exchange_bittrex.j2 | 10 --------- .../templates/subtemplates/exchange_kraken.j2 | 22 ++----------------- 4 files changed, 4 insertions(+), 47 deletions(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 780ad4161..faa8a98f4 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -61,13 +61,13 @@ def ask_user_config() -> Dict[str, Any]: "type": "text", "name": "stake_currency", "message": "Please insert your stake currency:", - "default": 'BTC', + "default": 'USDT', }, { "type": "text", "name": "stake_amount", "message": f"Please insert your stake amount (Number or '{UNLIMITED_STAKE_AMOUNT}'):", - "default": "0.01", + "default": "100", "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val), "filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"' if val == UNLIMITED_STAKE_AMOUNT diff --git a/freqtrade/templates/subtemplates/exchange_binance.j2 b/freqtrade/templates/subtemplates/exchange_binance.j2 index 38ba4fa5c..217002a7c 100644 --- a/freqtrade/templates/subtemplates/exchange_binance.j2 +++ b/freqtrade/templates/subtemplates/exchange_binance.j2 @@ -8,21 +8,6 @@ "rateLimit": 200 }, "pair_whitelist": [ - "ALGO/BTC", - "ATOM/BTC", - "BAT/BTC", - "BCH/BTC", - "BRD/BTC", - "EOS/BTC", - "ETH/BTC", - "IOTA/BTC", - "LINK/BTC", - "LTC/BTC", - "NEO/BTC", - "NXS/BTC", - "XMR/BTC", - "XRP/BTC", - "XTZ/BTC" ], "pair_blacklist": [ "BNB/BTC", diff --git a/freqtrade/templates/subtemplates/exchange_bittrex.j2 b/freqtrade/templates/subtemplates/exchange_bittrex.j2 index 7b27318ca..0394790ce 100644 --- a/freqtrade/templates/subtemplates/exchange_bittrex.j2 +++ b/freqtrade/templates/subtemplates/exchange_bittrex.j2 @@ -15,16 +15,6 @@ "rateLimit": 500 }, "pair_whitelist": [ - "ETH/BTC", - "LTC/BTC", - "ETC/BTC", - "DASH/BTC", - "ZEC/BTC", - "XLM/BTC", - "XRP/BTC", - "TRX/BTC", - "ADA/BTC", - "XMR/BTC" ], "pair_blacklist": [ ] diff --git a/freqtrade/templates/subtemplates/exchange_kraken.j2 b/freqtrade/templates/subtemplates/exchange_kraken.j2 index 7139a0830..4d0e4c1ff 100644 --- a/freqtrade/templates/subtemplates/exchange_kraken.j2 +++ b/freqtrade/templates/subtemplates/exchange_kraken.j2 @@ -7,28 +7,10 @@ "ccxt_async_config": { "enableRateLimit": true, "rateLimit": 1000 + // Enable the below for downoading data. + //"rateLimit": 3100 }, "pair_whitelist": [ - "ADA/EUR", - "ATOM/EUR", - "BAT/EUR", - "BCH/EUR", - "BTC/EUR", - "DAI/EUR", - "DASH/EUR", - "EOS/EUR", - "ETC/EUR", - "ETH/EUR", - "LINK/EUR", - "LTC/EUR", - "QTUM/EUR", - "REP/EUR", - "WAVES/EUR", - "XLM/EUR", - "XMR/EUR", - "XRP/EUR", - "XTZ/EUR", - "ZEC/EUR" ], "pair_blacklist": [ From c489e6825c3cc0affeb46e2a2f1f78d119e933f5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Sep 2021 09:18:15 +0200 Subject: [PATCH 024/239] Simplify binance blacklist --- .../templates/subtemplates/exchange_binance.j2 | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/freqtrade/templates/subtemplates/exchange_binance.j2 b/freqtrade/templates/subtemplates/exchange_binance.j2 index 217002a7c..3022464c7 100644 --- a/freqtrade/templates/subtemplates/exchange_binance.j2 +++ b/freqtrade/templates/subtemplates/exchange_binance.j2 @@ -10,17 +10,6 @@ "pair_whitelist": [ ], "pair_blacklist": [ - "BNB/BTC", - "BNB/BUSD", - "BNB/ETH", - "BNB/EUR", - "BNB/NGN", - "BNB/PAX", - "BNB/RUB", - "BNB/TRY", - "BNB/TUSD", - "BNB/USDC", - "BNB/USDS", - "BNB/USDT" + "BNB/.*", ] } From b4130dfabbdd0a68119080f33c990b34d4116d77 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Sep 2021 09:23:26 +0200 Subject: [PATCH 025/239] Use volumePairlist instead of staticPairlist in generated config --- freqtrade/templates/base_config.json.j2 | 9 ++++++++- freqtrade/templates/subtemplates/exchange_binance.j2 | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index a5782f7cd..68eebdbd4 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -1,3 +1,10 @@ +{%set volume_pairlist = '{ + "method": "VolumePairList", + "number_assets": 20, + "sort_key": "quoteVolume", + "min_value": 0, + "refresh_period": 1800 + }' %} { "max_open_trades": {{ max_open_trades }}, "stake_currency": "{{ stake_currency }}", @@ -29,7 +36,7 @@ }, {{ exchange | indent(4) }}, "pairlists": [ - {"method": "StaticPairList"} + {{ '{"method": "StaticPairList"}' if exchange_name == 'bittrex' else volume_pairlist }} ], "edge": { "enabled": false, diff --git a/freqtrade/templates/subtemplates/exchange_binance.j2 b/freqtrade/templates/subtemplates/exchange_binance.j2 index 3022464c7..de58b6f72 100644 --- a/freqtrade/templates/subtemplates/exchange_binance.j2 +++ b/freqtrade/templates/subtemplates/exchange_binance.j2 @@ -10,6 +10,6 @@ "pair_whitelist": [ ], "pair_blacklist": [ - "BNB/.*", + "BNB/.*" ] } From 2f92838c39639e4770b95258ca0357d4e69c7d40 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Sep 2021 16:18:47 +0200 Subject: [PATCH 026/239] Properly close parenteses in exchange doc --- docs/exchanges.md | 5 ++++- mkdocs.yml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index 81ad670a3..42a850acd 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -6,7 +6,6 @@ This page combines common gotchas and informations which are exchange-specific a Binance supports [time_in_force](configuration.md#understand-order_time_in_force). - !!! Tip "Stoploss on Exchange" Binance supports `stoploss_on_exchange` and uses stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it. @@ -116,6 +115,8 @@ Kucoin requires a passphrase for each api key, you will therefore need to add th "key": "your_exchange_key", "secret": "your_exchange_secret", "password": "your_exchange_api_key_password", + // ... +} ``` Kucoin supports [time_in_force](configuration.md#understand-order_time_in_force). @@ -163,6 +164,8 @@ For example, to test the order type `FOK` with Kraken, and modify candle limit t "order_time_in_force": ["gtc", "fok"], "ohlcv_candle_limit": 200 } + //... +} ``` !!! Warning diff --git a/mkdocs.yml b/mkdocs.yml index 854939ca0..05156168f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -23,10 +23,10 @@ nav: - Hyperopt: hyperopt.md - Utility Sub-commands: utils.md - Plotting: plotting.md + - Exchange-specific Notes: exchanges.md - Data Analysis: - Jupyter Notebooks: data-analysis.md - Strategy analysis: strategy_analysis_example.md - - Exchange-specific Notes: exchanges.md - Advanced Topics: - Advanced Post-installation Tasks: advanced-setup.md - Edge Positioning: edge.md From 103a8e827ea7e280b93d806d355ca1157ed55692 Mon Sep 17 00:00:00 2001 From: Rikj000 Date: Fri, 3 Sep 2021 16:37:36 +0200 Subject: [PATCH 027/239] =?UTF-8?q?=E2=9A=A1=20`setup.sh`=20-=20Use=20`bui?= =?UTF-8?q?ld=5Fhelpers/install=5Fta-lib.sh`=20for=20TA-Lib=20installation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build_helpers/install_ta-lib.sh | 2 +- setup.sh | 14 +------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/build_helpers/install_ta-lib.sh b/build_helpers/install_ta-lib.sh index dd87cf105..590b7d961 100755 --- a/build_helpers/install_ta-lib.sh +++ b/build_helpers/install_ta-lib.sh @@ -13,7 +13,7 @@ if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then && ./configure --prefix=${INSTALL_LOC}/ \ && make -j$(nproc) \ && which sudo && sudo make install || make install \ - && cd .. + && cd .. && rm -rf ./ta-lib/ else echo "TA-lib already installed, skipping installation" fi diff --git a/setup.sh b/setup.sh index e5f81578d..217500569 100755 --- a/setup.sh +++ b/setup.sh @@ -95,19 +95,7 @@ function install_talib() { return fi - cd build_helpers - tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib - sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h - ./configure --prefix=/usr/local - make - sudo make install - if [ -x "$(command -v apt-get)" ]; then - echo "Updating library path using ldconfig" - sudo ldconfig - fi - cd .. && rm -rf ./ta-lib/ - cd .. + cd build_helpers && ./install_ta-lib.sh && cd .. } function install_mac_newer_python_dependencies() { From 493fb3507316296f90f6c6384a1e84d366b82efa Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Sep 2021 22:05:40 +0200 Subject: [PATCH 028/239] Fix uvicorn not working properly on windows --- freqtrade/rpc/api_server/uvicorn_threaded.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/uvicorn_threaded.py b/freqtrade/rpc/api_server/uvicorn_threaded.py index b63999f51..79af659c7 100644 --- a/freqtrade/rpc/api_server/uvicorn_threaded.py +++ b/freqtrade/rpc/api_server/uvicorn_threaded.py @@ -5,6 +5,20 @@ import time import uvicorn +def asyncio_setup() -> None: # pragma: no cover + # Set eventloop for win32 setups + # Reverts a change done in uvicorn 0.15.0 - which now sets the eventloop + # via policy. + import sys + + if sys.version_info >= (3, 8) and sys.platform == "win32": + import asyncio + import selectors + selector = selectors.SelectSelector() + loop = asyncio.SelectorEventLoop(selector) + asyncio.set_event_loop(loop) + + class UvicornServer(uvicorn.Server): """ Multithreaded server - as found in https://github.com/encode/uvicorn/issues/742 @@ -28,7 +42,7 @@ class UvicornServer(uvicorn.Server): try: import uvloop # noqa except ImportError: # pragma: no cover - from uvicorn.loops.asyncio import asyncio_setup + asyncio_setup() else: asyncio.set_event_loop(uvloop.new_event_loop()) From eb0362c29ef7f125db4fda5590485c4c9f002f7b Mon Sep 17 00:00:00 2001 From: Rik Helsen Date: Fri, 3 Sep 2021 23:52:40 +0200 Subject: [PATCH 029/239] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20=EF=B8=8F`install?= =?UTF-8?q?=5Fta-lib.sh`=20-=20Run=20`ldconfig`=20after=20`make=20install`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build_helpers/install_ta-lib.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/build_helpers/install_ta-lib.sh b/build_helpers/install_ta-lib.sh index 590b7d961..d12b16364 100755 --- a/build_helpers/install_ta-lib.sh +++ b/build_helpers/install_ta-lib.sh @@ -12,9 +12,12 @@ if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub \ && ./configure --prefix=${INSTALL_LOC}/ \ && make -j$(nproc) \ - && which sudo && sudo make install || make install \ - && cd .. && rm -rf ./ta-lib/ + && which sudo && sudo make install || make install + if [ -x "$(command -v apt-get)" ]; then + echo "Updating library path using ldconfig" + sudo ldconfig + fi + cd .. && rm -rf ./ta-lib/ else echo "TA-lib already installed, skipping installation" fi -# && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \ From 4e049f65f227c7e2bf103db5cd9f97988c7a2250 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Sep 2021 09:14:44 +0200 Subject: [PATCH 030/239] Exclude some parts from coverage that can't really be tested --- freqtrade/__init__.py | 2 +- freqtrade/loggers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index e96e7f530..2747efc96 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -22,7 +22,7 @@ if __version__ == 'develop': # subprocess.check_output( # ['git', 'log', '--format="%h"', '-n 1'], # stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"') - except Exception: + except Exception: # pragma: no cover # git not available, ignore try: # Try Fallback to freqtrade_commit file (created by CI while building docker image) diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py index fbb05d879..5c5831695 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers.py @@ -87,7 +87,7 @@ def setup_logging(config: Dict[str, Any]) -> None: # syslog config. The messages should be equal for this. handler_sl.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s')) logging.root.addHandler(handler_sl) - elif s[0] == 'journald': + elif s[0] == 'journald': # pragma: no cover try: from systemd.journal import JournaldLogHandler except ImportError: From 2173ff0133db834cbda72a2d35fa6a07e5d574b9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Sep 2021 09:16:19 +0200 Subject: [PATCH 031/239] Update PR template to not link to issues in changelog --- .github/PULL_REQUEST_TEMPLATE.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 20ef27f0f..7c0655b20 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,14 +2,16 @@ Thank you for sending your pull request. But first, have you included unit tests, and is your code PEP8 conformant? [More details](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) ## Summary + Explain in one sentence the goal of this PR Solve the issue: #___ ## Quick changelog -- -- +- +- ## What's new? + *Explain in details what this PR solve or improve. You can include visuals.* From a8f28ffb11a48ce1a8bbb26b209bfd9fa34705d0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Sep 2021 15:34:57 +0200 Subject: [PATCH 032/239] Increase test coverage --- tests/strategy/test_interface.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 128599668..250dcf63d 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -735,11 +735,16 @@ def test_auto_hyperopt_interface(default_conf): PairLocks.timeframe = default_conf['timeframe'] strategy = StrategyResolver.load_strategy(default_conf) + with pytest.raises(OperationalException): + next(strategy.enumerate_parameters('deadBeef')) + assert strategy.buy_rsi.value == strategy.buy_params['buy_rsi'] # PlusDI is NOT in the buy-params, so default should be used assert strategy.buy_plusdi.value == 0.5 assert strategy.sell_rsi.value == strategy.sell_params['sell_rsi'] + assert repr(strategy.sell_rsi) == 'IntParameter(74)' + # Parameter is disabled - so value from sell_param dict will NOT be used. assert strategy.sell_minusdi.value == 0.5 all_params = strategy.detect_all_parameters() From c519ecf8df2fae0e8d789c5abacb6662620d73e7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Sep 2021 15:40:21 +0200 Subject: [PATCH 033/239] Exclude more untestable sections from coverage --- freqtrade/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 2fd3d32bb..6593fbcb6 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -9,7 +9,7 @@ from typing import Any, List # check min. python version -if sys.version_info < (3, 7): +if sys.version_info < (3, 7): # pragma: no cover sys.exit("Freqtrade requires Python version >= 3.7") from freqtrade.commands import Arguments @@ -46,7 +46,7 @@ def main(sysargv: List[str] = None) -> None: "`freqtrade --help` or `freqtrade --help`." ) - except SystemExit as e: + except SystemExit as e: # pragma: no cover return_code = e except KeyboardInterrupt: logger.info('SIGINT received, aborting ...') @@ -60,5 +60,5 @@ def main(sysargv: List[str] = None) -> None: sys.exit(return_code) -if __name__ == '__main__': +if __name__ == '__main__': # pragma: no cover main() From 1d24d3d5ee68624f2706e2a45ea939c87fa0b3ab Mon Sep 17 00:00:00 2001 From: lenik terenin Date: Sun, 5 Sep 2021 22:41:58 +0900 Subject: [PATCH 034/239] case insensitive blacklist Allow "btc/usdt" pairs in blacklist to match to "BTC/USDT" pairs that come from the exchange. --- freqtrade/plugins/pairlist/pairlist_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/pairlist_helpers.py b/freqtrade/plugins/pairlist/pairlist_helpers.py index 924bfb293..1de27fcbd 100644 --- a/freqtrade/plugins/pairlist/pairlist_helpers.py +++ b/freqtrade/plugins/pairlist/pairlist_helpers.py @@ -17,7 +17,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], if keep_invalid: for pair_wc in wildcardpl: try: - comp = re.compile(pair_wc) + comp = re.compile(pair_wc, re.IGNORECASE) result_partial = [ pair for pair in available_pairs if re.fullmatch(comp, pair) ] @@ -33,7 +33,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], else: for pair_wc in wildcardpl: try: - comp = re.compile(pair_wc) + comp = re.compile(pair_wc, re.IGNORECASE) result += [ pair for pair in available_pairs if re.fullmatch(comp, pair) ] From 4daa4b9e63960da073e8e2d5f7e00906d4117cf5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Sep 2021 03:01:10 +0000 Subject: [PATCH 035/239] Bump pytest from 6.2.4 to 6.2.5 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.2.4 to 6.2.5. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.2.4...6.2.5) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 67ee0035b..34d5607f3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ flake8==3.9.2 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.4.1 mypy==0.910 -pytest==6.2.4 +pytest==6.2.5 pytest-asyncio==0.15.1 pytest-cov==2.12.1 pytest-mock==3.6.1 From 771193cbe47b0bd812e87662bdc141b911e316a7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Sep 2021 03:01:19 +0000 Subject: [PATCH 036/239] Bump plotly from 5.3.0 to 5.3.1 Bumps [plotly](https://github.com/plotly/plotly.py) from 5.3.0 to 5.3.1. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v5.3.0...v5.3.1) --- updated-dependencies: - dependency-name: plotly dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index 62836a729..8e17232b0 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.3.0 +plotly==5.3.1 From 44f8d7abf2ba21c3eb0e1c0f5804d8bafb9d2e31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Sep 2021 03:01:23 +0000 Subject: [PATCH 037/239] Bump ccxt from 1.55.56 to 1.55.83 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.55.56 to 1.55.83. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.55.56...1.55.83) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f77edddfe..e2bed0f9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.21.2 pandas==1.3.2 -ccxt==1.55.56 +ccxt==1.55.83 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.8 aiohttp==3.7.4.post0 From 765e72715bf739504e53140cd29eb24ef40dda44 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Sep 2021 03:01:28 +0000 Subject: [PATCH 038/239] Bump mkdocs-material from 7.2.5 to 7.2.6 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 7.2.5 to 7.2.6. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/docs/changelog.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/7.2.5...7.2.6) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index d820c9412..9927740c2 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.2 -mkdocs-material==7.2.5 +mkdocs-material==7.2.6 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 From a04875eb55beab2f9d86534434daf2465b9f0d78 Mon Sep 17 00:00:00 2001 From: EnzovdWetering <36194995+EnzovdWetering@users.noreply.github.com> Date: Mon, 6 Sep 2021 17:53:44 +0200 Subject: [PATCH 039/239] Update edge.md Typo fix --- docs/edge.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/edge.md b/docs/edge.md index 237ff36f6..4402d767f 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -3,7 +3,7 @@ The `Edge Positioning` module uses probability to calculate your win rate and risk reward ratio. It will use these statistics to control your strategy trade entry points, position size and, stoploss. !!! Warning - WHen using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data. + When using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data. !!! Note `Edge Positioning` only considers *its own* buy/sell/stoploss signals. It ignores the stoploss, trailing stoploss, and ROI settings in the strategy configuration file. From 6bd495a32a58b6da236f53da2d166f683a21cfff Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Sep 2021 19:53:32 +0200 Subject: [PATCH 040/239] Fix 0Exception error happens when wrong stake-currency is selected and /profit is called --- freqtrade/rpc/rpc.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 95a37452b..ca2e84e48 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -403,8 +403,11 @@ class RPC: # Doing the sum is not right - overall profit needs to be based on initial capital profit_all_ratio_sum = sum(profit_all_ratio) if profit_all_ratio else 0.0 starting_balance = self._freqtrade.wallets.get_starting_balance() - profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance - profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance + profit_closed_ratio_fromstart = 0 + profit_all_ratio_fromstart = 0 + if starting_balance: + profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance + profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance profit_all_fiat = self._fiat_converter.convert_amount( profit_all_coin_sum, From 10d0987f49b8179f840895882bccebb732098192 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Sep 2021 19:54:34 +0200 Subject: [PATCH 041/239] Fix docs for custom hyperopt space --- docs/advanced-hyperopt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 8f233438b..4a2bafd2e 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -80,7 +80,7 @@ To override a pre-defined space (`roi_space`, `generate_roi_table`, `stoploss_sp class MyAwesomeStrategy(IStrategy): class HyperOpt: # Define a custom stoploss space. - def stoploss_space(self): + def stoploss_space(): return [SKDecimal(-0.05, -0.01, decimals=3, name='stoploss')] ``` From 880474594e5357eea9b47b8d1c8449a968bce4c3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Sep 2021 06:51:31 +0200 Subject: [PATCH 042/239] have ftuser use `/bin/bash` in dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4c4722452..f7e26efe3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ RUN mkdir /freqtrade \ && apt-get update \ && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \ && apt-get clean \ - && useradd -u 1000 -G sudo -U -m ftuser \ + && useradd -u 1000 -G sudo -U -m -s /bin/bash ftuser \ && chown ftuser:ftuser /freqtrade \ # Allow sudoers && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers From 71ff214adfe844a1ff57624094169f48dbd202e5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Sep 2021 07:14:40 +0200 Subject: [PATCH 043/239] Support "initial_call" for download-data of new pairs --- freqtrade/data/history/history_utils.py | 3 ++- freqtrade/exchange/binance.py | 1 + freqtrade/exchange/exchange.py | 17 ++++++++++++----- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 6f125aaa9..e6b8db322 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -197,7 +197,8 @@ def _download_pair_history(pair: str, *, timeframe=timeframe, since_ms=since_ms if since_ms else arrow.utcnow().shift( - days=-new_pairs_days).int_timestamp * 1000 + days=-new_pairs_days).int_timestamp * 1000, + is_new_pair=data.empty ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 189f5f481..7a3111f85 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -20,6 +20,7 @@ class Binance(Exchange): "order_time_in_force": ['gtc', 'fok', 'ioc'], "time_in_force_parameter": "timeInForce", "ohlcv_candle_limit": 1000, + "ohlcv_initial_call": True, "trades_pagination": "id", "trades_pagination_arg": "fromId", "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000], diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 80f20b17e..425b89f88 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1194,7 +1194,7 @@ class Exchange: # Historic data def get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int) -> List: + since_ms: int, is_new_pair: bool = False) -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1206,7 +1206,7 @@ class Exchange: """ return asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms)) + since_ms=since_ms, is_new_pair=is_new_pair)) def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, since_ms: int) -> DataFrame: @@ -1221,9 +1221,9 @@ class Exchange: return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) - async def _async_get_historic_ohlcv(self, pair: str, - timeframe: str, - since_ms: int) -> List: + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool + ) -> List: """ Download historic ohlcv """ @@ -1234,6 +1234,13 @@ class Exchange: one_call, arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) ) + if self._ft_has.get('ohlcv_initial_call', False) and is_new_pair: + x = await self._async_get_candle_history(pair, timeframe, 0) + if x and x[2] and x[2][0] and x[2][0][0] > since_ms: + # Set starting date to first available candle. + since_ms = x[2][0][0] + logger.info(f"Candle-data available starting with {since_ms}.") + input_coroutines = [self._async_get_candle_history( pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] From 8c83c258a52c9ea4d91ed8f0d1a87e084109f828 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Sep 2021 19:29:10 +0200 Subject: [PATCH 044/239] Move "first-pair_getting" to binance subclass --- freqtrade/exchange/binance.py | 21 +++++++++++++++++++-- freqtrade/exchange/exchange.py | 8 +------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 7a3111f85..284f9cdad 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -1,7 +1,8 @@ """ Binance exchange subclass """ import logging -from typing import Dict +from typing import Dict, List +import arrow import ccxt from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, @@ -20,7 +21,6 @@ class Binance(Exchange): "order_time_in_force": ['gtc', 'fok', 'ioc'], "time_in_force_parameter": "timeInForce", "ohlcv_candle_limit": 1000, - "ohlcv_initial_call": True, "trades_pagination": "id", "trades_pagination_arg": "fromId", "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000], @@ -91,3 +91,20 @@ class Binance(Exchange): f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: raise OperationalException(e) from e + + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool + ) -> List: + """ + Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date + Does not work for other exchanges, which don't return the earliest data when called with "0" + """ + if is_new_pair: + x = await self._async_get_candle_history(pair, timeframe, 0) + if x and x[2] and x[2][0] and x[2][0][0] > since_ms: + # Set starting date to first available candle. + since_ms = x[2][0][0] + logger.info(f"Candle-data available starting with " + f"{arrow.get(since_ms // 1000).isoformat()}.") + return await super()._async_get_historic_ohlcv( + pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 425b89f88..79fd33dfe 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1226,6 +1226,7 @@ class Exchange: ) -> List: """ Download historic ohlcv + :param is_new_pair: used by binance subclass to allow "fast" new pair downloading """ one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) @@ -1234,13 +1235,6 @@ class Exchange: one_call, arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) ) - if self._ft_has.get('ohlcv_initial_call', False) and is_new_pair: - x = await self._async_get_candle_history(pair, timeframe, 0) - if x and x[2] and x[2][0] and x[2][0][0] > since_ms: - # Set starting date to first available candle. - since_ms = x[2][0][0] - logger.info(f"Candle-data available starting with {since_ms}.") - input_coroutines = [self._async_get_candle_history( pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] From 2d66987ac7ed38961c2110680b03893693f86231 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Sep 2021 19:51:07 +0200 Subject: [PATCH 045/239] Add test for "pair-startdate" detection --- freqtrade/exchange/binance.py | 2 +- tests/exchange/test_binance.py | 35 +++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 284f9cdad..8dced3894 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -104,7 +104,7 @@ class Binance(Exchange): if x and x[2] and x[2][0] and x[2][0][0] > since_ms: # Set starting date to first available candle. since_ms = x[2][0][0] - logger.info(f"Candle-data available starting with " + logger.info(f"Candle-data for {pair} available starting with " f"{arrow.get(since_ms // 1000).isoformat()}.") return await super()._async_get_historic_ohlcv( pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair) diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index f2b508761..dd85c3abe 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone from random import randint from unittest.mock import MagicMock @@ -5,7 +6,7 @@ import ccxt import pytest from freqtrade.exceptions import DependencyException, InvalidOrderException, OperationalException -from tests.conftest import get_patched_exchange +from tests.conftest import get_mock_coro, get_patched_exchange, log_has_re from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -105,3 +106,35 @@ def test_stoploss_adjust_binance(mocker, default_conf): # Test with invalid order case order['type'] = 'stop_loss' assert not exchange.stoploss_adjust(1501, order) + + +@pytest.mark.asyncio +async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog): + ohlcv = [ + [ + int((datetime.now(timezone.utc).timestamp() - 1000) * 1000), + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + exchange = get_patched_exchange(mocker, default_conf, id='binance') + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) + + pair = 'ETH/BTC' + res = await exchange._async_get_historic_ohlcv(pair, "5m", + 1500000000000, is_new_pair=False) + # Call with very old timestamp - causes tons of requests + assert exchange._api_async.fetch_ohlcv.call_count > 400 + # assert res == ohlcv + exchange._api_async.fetch_ohlcv.reset_mock() + res = await exchange._async_get_historic_ohlcv(pair, "5m", 1500000000000, is_new_pair=True) + + # Called twice - one "init" call - and one to get the actual data. + assert exchange._api_async.fetch_ohlcv.call_count == 2 + assert res == ohlcv + assert log_has_re(r"Candle-data for ETH/BTC available starting with .*", caplog) From 79ca6135a2cb599de9fa4fde397226d63dda2832 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Tue, 7 Sep 2021 21:53:38 -0600 Subject: [PATCH 046/239] added caplog clears to freqtradebot tests --- tests/test_freqtradebot.py | 53 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 75b67e59c..ddd031f77 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -107,6 +107,7 @@ def test_order_dict_dry_run(default_conf, mocker, caplog) -> None: freqtrade = FreqtradeBot(conf) assert not freqtrade.strategy.order_types['stoploss_on_exchange'] assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) + caplog.clear() def test_order_dict_live(default_conf, mocker, caplog) -> None: @@ -140,6 +141,7 @@ def test_order_dict_live(default_conf, mocker, caplog) -> None: freqtrade = FreqtradeBot(conf) assert not freqtrade.strategy.order_types['stoploss_on_exchange'] assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) + caplog.clear() def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None: @@ -415,6 +417,7 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord assert freqtrade.create_trade('ETH/BTC') assert log_has_re(r"Stake amount for pair .* is too small.*", caplog) + caplog.clear() def test_create_trade_zero_stake_amount(default_conf, ticker, limit_buy_order_open, @@ -478,6 +481,7 @@ def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_ope n = freqtrade.enter_positions() assert n == 0 assert log_has_re(r"No currency pair in active pair whitelist.*", caplog) + caplog.clear() def test_enter_positions_no_pairs_in_whitelist(default_conf, ticker, limit_buy_order, fee, @@ -518,11 +522,13 @@ def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order, # 0 trades, but it's not because of pairlock. assert n == 0 assert not log_has_re(message, caplog) + caplog.clear() PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because') n = freqtrade.enter_positions() assert n == 0 assert log_has_re(message, caplog) + caplog.clear() def test_create_trade_no_signal(default_conf, fee, mocker) -> None: @@ -1086,6 +1092,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, assert log_has_re(r'STOP_LOSS_LIMIT is hit for Trade\(id=1, .*\)\.', caplog) assert trade.stoploss_order_id is None assert trade.is_open is False + caplog.clear() mocker.patch( 'freqtrade.exchange.Binance.stoploss', @@ -1115,6 +1122,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) assert freqtrade.handle_stoploss_on_exchange(trade) is False assert stoploss.call_count == 0 + caplog.clear() def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, @@ -1154,6 +1162,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, assert log_has_re(r'Stoploss order was cancelled, but unable to recreate one.*', caplog) assert trade.stoploss_order_id is None assert trade.is_open is True + caplog.clear() def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, @@ -1202,6 +1211,7 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, assert rpc_mock.call_count == 2 assert rpc_mock.call_args_list[1][0][0]['sell_reason'] == SellType.EMERGENCY_SELL.value assert rpc_mock.call_args_list[1][0][0]['order_type'] == 'market' + caplog.clear() def test_create_stoploss_order_insufficient_funds(mocker, default_conf, caplog, fee, @@ -1428,6 +1438,7 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging) assert cancel_mock.call_count == 1 assert log_has_re(r"Could not create trailing stoploss order for pair ETH/BTC\..*", caplog) + caplog.clear() @pytest.mark.usefixtures("init_persistence") @@ -1662,6 +1673,7 @@ def test_enter_positions(mocker, default_conf, caplog) -> None: assert log_has('Found no buy signals for whitelisted currencies. Trying again...', caplog) # create_trade should be called once for every pair in the whitelist. assert mock_ct.call_count == len(default_conf['exchange']['pair_whitelist']) + caplog.clear() def test_enter_positions_exception(mocker, default_conf, caplog) -> None: @@ -1701,6 +1713,7 @@ def test_exit_positions(mocker, default_conf, limit_buy_order, caplog) -> None: # test amount modified by fee-logic n = freqtrade.exit_positions(trades) assert n == 0 + caplog.clear() def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog) -> None: @@ -1721,6 +1734,7 @@ def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog) n = freqtrade.exit_positions(trades) assert n == 0 assert log_has('Unable to sell trade ETH/BTC: ', caplog) + caplog.clear() def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> None: @@ -1743,10 +1757,12 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No ) assert not freqtrade.update_trade_state(trade, None) assert log_has_re(r'Orderid for trade .* is empty.', caplog) + caplog.clear() # Add datetime explicitly since sqlalchemy defaults apply only once written to database freqtrade.update_trade_state(trade, '123') # Test amount not modified by fee-logic assert not log_has_re(r'Applying fee to .*', caplog) + caplog.clear() assert trade.open_order_id is None assert trade.amount == limit_buy_order['amount'] @@ -1764,6 +1780,7 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No freqtrade.update_trade_state(trade, '123') assert log_has_re('Found open order for.*', caplog) + caplog.clear() def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_buy_order, fee, @@ -1814,6 +1831,7 @@ def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_ assert trade.amount != amount assert trade.amount == limit_buy_order['amount'] assert log_has_re(r'Applying fee on amount for .*', caplog) + caplog.clear() def test_update_trade_state_exception(mocker, default_conf, @@ -1832,6 +1850,7 @@ def test_update_trade_state_exception(mocker, default_conf, ) freqtrade.update_trade_state(trade, trade.open_order_id) assert log_has('Could not update trade amount: ', caplog) + caplog.clear() def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None: @@ -1848,6 +1867,7 @@ def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None freqtrade.update_trade_state(trade, trade.open_order_id) assert grm_mock.call_count == 0 assert log_has(f'Unable to fetch order {trade.open_order_id}: ', caplog) + caplog.clear() def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_order_open, @@ -2016,6 +2036,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order_open, assert freqtrade.handle_trade(trade) assert log_has("ETH/BTC - Required profit reached. sell_type=SellType.ROI", caplog) + caplog.clear() def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_open, @@ -2048,6 +2069,7 @@ def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_open assert freqtrade.handle_trade(trade) assert log_has("ETH/BTC - Sell signal received. sell_type=SellType.SELL_SIGNAL", caplog) + caplog.clear() def test_close_trade(default_conf, ticker, limit_buy_order, limit_buy_order_open, limit_sell_order, @@ -2088,6 +2110,7 @@ def test_bot_loop_start_called_once(mocker, default_conf, caplog): assert log_has_re(r'Strategy caused the following exception.*', caplog) assert ftbot.strategy.bot_loop_start.call_count == 1 assert ftbot.strategy.analyze.call_count == 1 + caplog.clear() def test_check_handle_timedout_buy_usercustom(default_conf, ticker, limit_buy_order_old, open_trade, @@ -2202,6 +2225,7 @@ def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, o nb_trades = len(trades) assert nb_trades == 0 assert log_has_re("Buy order cancelled on exchange for Trade.*", caplog) + caplog.clear() def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_order_old, open_trade, @@ -2336,6 +2360,7 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old, assert rpc_mock.call_count == 1 assert open_trade.is_open is True assert log_has_re("Sell order cancelled on exchange for Trade.*", caplog) + caplog.clear() def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, @@ -2404,6 +2429,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap assert trades[0].open_order_id is None assert trades[0].fee_updated('buy') assert pytest.approx(trades[0].fee_open) == 0.001 + caplog.clear() def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, caplog, fee, @@ -2444,6 +2470,7 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, limit_buy_order_old_partial['remaining']) assert trades[0].open_order_id is None assert trades[0].fee_open == fee() + caplog.clear() def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocker, caplog) -> None: @@ -2472,6 +2499,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke f"{open_trade.open_date.strftime('%Y-%m-%d %H:%M:%S')}" r"\) due to Traceback \(most recent call last\):\n*", caplog) + caplog.clear() def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> None: @@ -2515,6 +2543,7 @@ def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> Non mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock) assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) assert log_has_re(r"Order .* for .* not cancelled.", caplog) + caplog.clear() @pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'ftx', 'kraken', 'bittrex'], @@ -2536,6 +2565,7 @@ def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf, assert cancel_order_mock.call_count == 0 assert log_has_re(r'Buy order fully cancelled. Removing .* from database\.', caplog) assert nofiy_mock.call_count == 1 + caplog.clear() @pytest.mark.parametrize('cancelorder', [ @@ -2905,6 +2935,7 @@ def test_execute_trade_exit_sloe_cancel_exception( sell_reason=SellCheckTuple(sell_type=SellType.STOP_LOSS)) assert create_order_mock.call_count == 2 assert log_has('Could not cancel stoploss order abcd', caplog) + caplog.clear() def test_execute_trade_exit_with_stoploss_on_exchange(default_conf, ticker, fee, ticker_sell_up, @@ -3300,6 +3331,7 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_ assert freqtrade.handle_trade(trade) is True assert log_has_re(r'.*Falling back to wallet-amount.', caplog) assert trade.amount != amnt + caplog.clear() def test__safe_sell_amount(default_conf, fee, caplog, mocker): @@ -3330,6 +3362,7 @@ def test__safe_sell_amount(default_conf, fee, caplog, mocker): assert freqtrade._safe_sell_amount(trade.pair, amount_wallet) == amount_wallet assert not log_has_re(r'.*Falling back to wallet-amount.', caplog) assert wallet_update.call_count == 1 + caplog.clear() def test__safe_sell_amount_error(default_conf, fee, caplog, mocker): @@ -3386,6 +3419,7 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo freqtrade.enter_positions() assert log_has_re(f"Pair {trade.pair} is still locked.*", caplog) + caplog.clear() def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, @@ -3477,6 +3511,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order_open, limit_buy_order, assert log_has("ETH/BTC - HIT STOP: current price at 0.000012, stoploss is 0.000015, " "initial stoploss was at 0.000010, trade opened at 0.000011", caplog) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value + caplog.clear() def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_order_open, fee, @@ -3524,6 +3559,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_or assert log_has("ETH/BTC - Using positive stoploss: 0.01 offset: 0 profit: 0.2666%", caplog) assert log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000138501 + caplog.clear() mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ @@ -3537,6 +3573,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_or f"ETH/BTC - HIT STOP: current price at {buy_price + 0.000002:.6f}, " f"stoploss is {trade.stop_loss:.6f}, " f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog) + caplog.clear() def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_order_open, fee, @@ -3584,6 +3621,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_orde assert log_has("ETH/BTC - Using positive stoploss: 0.01 offset: 0.011 profit: 0.2666%", caplog) assert log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000138501 + caplog.clear() mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ @@ -3598,6 +3636,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_orde f"stoploss is {trade.stop_loss:.6f}, " f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value + caplog.clear() def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_open, fee, @@ -3648,6 +3687,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_ assert not log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000098910 + caplog.clear() # price rises above the offset (rises 12% when the offset is 5.5%) mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', @@ -3661,6 +3701,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_ assert log_has("ETH/BTC - Using positive stoploss: 0.05 offset: 0.055 profit: 0.1218%", caplog) assert log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000117705 + caplog.clear() def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, @@ -3722,6 +3763,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).', caplog) + caplog.clear() def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fee, fee, @@ -3747,6 +3789,7 @@ def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fe assert walletmock.call_count == 1 assert log_has_re(r'Fee amount for Trade.* was in base currency ' '- Eating Fee 0.008 into dust', caplog) + caplog.clear() def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, fee): @@ -3769,6 +3812,7 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' 'open_rate=0.24544100, open_since=closed) failed: myTrade-Dict empty found', caplog) + caplog.clear() def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fee, mocker): @@ -3862,6 +3906,7 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c assert trade.fee_open_currency is not None assert trade.fee_close_cost is None assert trade.fee_close_currency is None + caplog.clear() def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, caplog, fee, @@ -3897,6 +3942,7 @@ def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, assert trade.fee_open_currency is not None assert trade.fee_close_cost is None assert trade.fee_close_currency is None + caplog.clear() def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, fee, @@ -3925,6 +3971,7 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).', caplog) + caplog.clear() def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, fee, mocker): @@ -4168,6 +4215,7 @@ def test_order_book_bid_strategy_exception(mocker, default_conf, caplog) -> None with pytest.raises(PricingError): freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") assert log_has_re(r'Buy Price at location 1 from orderbook could not be determined.', caplog) + caplog.clear() def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: @@ -4238,6 +4286,7 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order_open, limit_buy_o freqtrade.handle_trade(trade) assert log_has_re(r'Sell Price at location 1 from orderbook could not be determined\..*', caplog) + caplog.clear() def test_startup_state(default_conf, mocker): @@ -4296,6 +4345,7 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order_ assert log_has_re(r"Unable to create trade for XRP/BTC: " r"Available balance \(0.0 BTC\) is lower than stake amount \(0.001 BTC\)", caplog) + caplog.clear() @pytest.mark.usefixtures("init_persistence") @@ -4339,6 +4389,7 @@ def test_update_open_orders(mocker, default_conf, fee, caplog): freqtrade.update_open_orders() assert not log_has_re(r"Error updating Order .*", caplog) + caplog.clear() freqtrade.config['dry_run'] = False freqtrade.update_open_orders() @@ -4454,6 +4505,7 @@ def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): assert log_has_re(r"Trying to reupdate buy fees for .*", caplog) assert mock_uts.call_count == 0 assert not log_has_re(r"Updating buy-fee on trade .* for order .*\.", caplog) + caplog.clear() @pytest.mark.usefixtures("init_persistence") @@ -4591,6 +4643,7 @@ def test_refind_lost_order(mocker, default_conf, fee, caplog): freqtrade.refind_lost_order(trades[4]) assert log_has(f"Error updating {order['id']}.", caplog) + caplog.clear() def test_get_valid_price(mocker, default_conf) -> None: From aac05029e1d13936f89c10f0ed8ccdaec3561ed7 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 01:20:52 -0600 Subject: [PATCH 047/239] safe_sell_amount -> safe_exit_amount --- freqtrade/freqtradebot.py | 4 ++-- tests/test_freqtradebot.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 259270483..0fc40cf45 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1039,7 +1039,7 @@ class FreqtradeBot(LoggingMixin): ) return reason - def _safe_sell_amount(self, pair: str, amount: float) -> float: + def _safe_exit_amount(self, pair: str, amount: float) -> float: """ Get sellable amount. Should be trade.amount - but will fall back to the available amount if necessary. @@ -1111,7 +1111,7 @@ class FreqtradeBot(LoggingMixin): # but we allow this value to be changed) order_type = self.strategy.order_types.get("forcesell", order_type) - amount = self._safe_sell_amount(trade.pair, trade.amount) + amount = self._safe_exit_amount(trade.pair, trade.amount) time_in_force = self.strategy.order_time_in_force['sell'] if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 75b67e59c..109cb01c2 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3302,7 +3302,7 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_ assert trade.amount != amnt -def test__safe_sell_amount(default_conf, fee, caplog, mocker): +def test__safe_exit_amount(default_conf, fee, caplog, mocker): patch_RPCManager(mocker) patch_exchange(mocker) amount = 95.33 @@ -3322,17 +3322,17 @@ def test__safe_sell_amount(default_conf, fee, caplog, mocker): patch_get_signal(freqtrade) wallet_update.reset_mock() - assert freqtrade._safe_sell_amount(trade.pair, trade.amount) == amount_wallet + assert freqtrade._safe_exit_amount(trade.pair, trade.amount) == amount_wallet assert log_has_re(r'.*Falling back to wallet-amount.', caplog) assert wallet_update.call_count == 1 caplog.clear() wallet_update.reset_mock() - assert freqtrade._safe_sell_amount(trade.pair, amount_wallet) == amount_wallet + assert freqtrade._safe_exit_amount(trade.pair, amount_wallet) == amount_wallet assert not log_has_re(r'.*Falling back to wallet-amount.', caplog) assert wallet_update.call_count == 1 -def test__safe_sell_amount_error(default_conf, fee, caplog, mocker): +def test__safe_exit_amount_error(default_conf, fee, caplog, mocker): patch_RPCManager(mocker) patch_exchange(mocker) amount = 95.33 @@ -3350,7 +3350,7 @@ def test__safe_sell_amount_error(default_conf, fee, caplog, mocker): freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) with pytest.raises(DependencyException, match=r"Not enough amount to sell."): - assert freqtrade._safe_sell_amount(trade.pair, trade.amount) + assert freqtrade._safe_exit_amount(trade.pair, trade.amount) def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplog) -> None: From be93c75e44fcaa1df2094b1d6200b0c884abf596 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 01:14:16 -0600 Subject: [PATCH 048/239] reupdate_buy_order_fees -> reupdate_enter_order_fees --- freqtrade/freqtradebot.py | 4 ++-- tests/test_freqtradebot.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0fc40cf45..0f16162cd 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -296,9 +296,9 @@ class FreqtradeBot(LoggingMixin): if sell_order: self.refind_lost_order(trade) else: - self.reupdate_buy_order_fees(trade) + self.reupdate_enter_order_fees(trade) - def reupdate_buy_order_fees(self, trade: Trade): + def reupdate_enter_order_fees(self, trade: Trade): """ Get buy order from database, and try to reupdate. Handles trades where the initial fee-update did not work. diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 109cb01c2..88ed11558 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -4420,14 +4420,14 @@ def test_update_closed_trades_without_assigned_fees(mocker, default_conf, fee): @pytest.mark.usefixtures("init_persistence") -def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): +def test_reupdate_enter_order_fees(mocker, default_conf, fee, caplog): freqtrade = get_patched_freqtradebot(mocker, default_conf) mock_uts = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.update_trade_state') create_mock_trades(fee) trades = Trade.get_trades().all() - freqtrade.reupdate_buy_order_fees(trades[0]) + freqtrade.reupdate_enter_order_fees(trades[0]) assert log_has_re(r"Trying to reupdate buy fees for .*", caplog) assert mock_uts.call_count == 1 assert mock_uts.call_args_list[0][0][0] == trades[0] @@ -4450,7 +4450,7 @@ def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): ) Trade.query.session.add(trade) - freqtrade.reupdate_buy_order_fees(trade) + freqtrade.reupdate_enter_order_fees(trade) assert log_has_re(r"Trying to reupdate buy fees for .*", caplog) assert mock_uts.call_count == 0 assert not log_has_re(r"Updating buy-fee on trade .* for order .*\.", caplog) @@ -4460,7 +4460,7 @@ def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): def test_handle_insufficient_funds(mocker, default_conf, fee): freqtrade = get_patched_freqtradebot(mocker, default_conf) mock_rlo = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.refind_lost_order') - mock_bof = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.reupdate_buy_order_fees') + mock_bof = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.reupdate_enter_order_fees') create_mock_trades(fee) trades = Trade.get_trades().all() From e0092a85e9769eeb7e73bbb6f6018dbd494ea1a0 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 00:53:09 -0600 Subject: [PATCH 049/239] handle_cancel_buy/sell -> handle_cancel_enter/exit --- freqtrade/freqtradebot.py | 12 +++++------ freqtrade/rpc/rpc.py | 4 ++-- tests/test_freqtradebot.py | 44 +++++++++++++++++++------------------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0f16162cd..bf07ac0bc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -906,7 +906,7 @@ class FreqtradeBot(LoggingMixin): default_retval=False)(pair=trade.pair, trade=trade, order=order))): - self.handle_cancel_buy(trade, order, constants.CANCEL_REASON['TIMEOUT']) + self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['TIMEOUT']) elif (order['side'] == 'sell' and (order['status'] == 'open' or fully_cancelled) and ( fully_cancelled @@ -915,7 +915,7 @@ class FreqtradeBot(LoggingMixin): default_retval=False)(pair=trade.pair, trade=trade, order=order))): - self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['TIMEOUT']) + self.handle_cancel_exit(trade, order, constants.CANCEL_REASON['TIMEOUT']) def cancel_all_open_orders(self) -> None: """ @@ -931,13 +931,13 @@ class FreqtradeBot(LoggingMixin): continue if order['side'] == 'buy': - self.handle_cancel_buy(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) elif order['side'] == 'sell': - self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + self.handle_cancel_exit(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) Trade.commit() - def handle_cancel_buy(self, trade: Trade, order: Dict, reason: str) -> bool: + def handle_cancel_enter(self, trade: Trade, order: Dict, reason: str) -> bool: """ Buy cancel - cancel order :return: True if order was fully cancelled @@ -998,7 +998,7 @@ class FreqtradeBot(LoggingMixin): reason=reason) return was_trade_fully_canceled - def handle_cancel_sell(self, trade: Trade, order: Dict, reason: str) -> str: + def handle_cancel_exit(self, trade: Trade, order: Dict, reason: str) -> str: """ Sell cancel - cancel order and update trade :return: Reason for cancel diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index ca2e84e48..caf7345a2 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -548,12 +548,12 @@ class RPC: order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair) if order['side'] == 'buy': - fully_canceled = self._freqtrade.handle_cancel_buy( + fully_canceled = self._freqtrade.handle_cancel_enter( trade, order, CANCEL_REASON['FORCE_SELL']) if order['side'] == 'sell': # Cancel order - so it is placed anew with a fresh price. - self._freqtrade.handle_cancel_sell(trade, order, CANCEL_REASON['FORCE_SELL']) + self._freqtrade.handle_cancel_exit(trade, order, CANCEL_REASON['FORCE_SELL']) if not fully_canceled: # Get current rate and execute sell diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 88ed11558..1b5381264 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2453,8 +2453,8 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', - handle_cancel_buy=MagicMock(), - handle_cancel_sell=MagicMock(), + handle_cancel_enter=MagicMock(), + handle_cancel_exit=MagicMock(), ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -2474,7 +2474,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke caplog) -def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> None: +def test_handle_cancel_enter(mocker, caplog, default_conf, limit_buy_order) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_buy_order = deepcopy(limit_buy_order) @@ -2493,34 +2493,34 @@ def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> Non limit_buy_order['filled'] = 0.0 limit_buy_order['status'] = 'open' reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() caplog.clear() limit_buy_order['filled'] = 0.01 - assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 0 assert log_has_re("Order .* for .* not cancelled, as the filled amount.* unsellable.*", caplog) caplog.clear() cancel_order_mock.reset_mock() limit_buy_order['filled'] = 2 - assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 # Order remained open for some reason (cancel failed) cancel_buy_order['status'] = 'open' cancel_order_mock = MagicMock(return_value=cancel_buy_order) mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock) - assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert log_has_re(r"Order .* for .* not cancelled.", caplog) @pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'ftx', 'kraken', 'bittrex'], indirect=['limit_buy_order_canceled_empty']) -def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf, - limit_buy_order_canceled_empty) -> None: +def test_handle_cancel_enter_exchanges(mocker, caplog, default_conf, + limit_buy_order_canceled_empty) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = mocker.patch( @@ -2532,7 +2532,7 @@ def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf, reason = CANCEL_REASON['TIMEOUT'] trade = MagicMock() trade.pair = 'LTC/ETH' - assert freqtrade.handle_cancel_buy(trade, limit_buy_order_canceled_empty, reason) + assert freqtrade.handle_cancel_enter(trade, limit_buy_order_canceled_empty, reason) assert cancel_order_mock.call_count == 0 assert log_has_re(r'Buy order fully cancelled. Removing .* from database\.', caplog) assert nofiy_mock.call_count == 1 @@ -2544,8 +2544,8 @@ def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf, 'String Return value', 123 ]) -def test_handle_cancel_buy_corder_empty(mocker, default_conf, limit_buy_order, - cancelorder) -> None: +def test_handle_cancel_enter_corder_empty(mocker, default_conf, limit_buy_order, + cancelorder) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock(return_value=cancelorder) @@ -2563,16 +2563,16 @@ def test_handle_cancel_buy_corder_empty(mocker, default_conf, limit_buy_order, limit_buy_order['filled'] = 0.0 limit_buy_order['status'] = 'open' reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() limit_buy_order['filled'] = 1.0 - assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 -def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None: +def test_handle_cancel_exit_limit(mocker, default_conf, fee) -> None: send_msg_mock = patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock() @@ -2598,26 +2598,26 @@ def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None: 'amount': 1, 'status': "open"} reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_sell(trade, order, reason) + assert freqtrade.handle_cancel_exit(trade, order, reason) assert cancel_order_mock.call_count == 1 assert send_msg_mock.call_count == 1 send_msg_mock.reset_mock() order['amount'] = 2 - assert freqtrade.handle_cancel_sell(trade, order, reason + assert freqtrade.handle_cancel_exit(trade, order, reason ) == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] # Assert cancel_order was not called (callcount remains unchanged) assert cancel_order_mock.call_count == 1 assert send_msg_mock.call_count == 1 - assert freqtrade.handle_cancel_sell(trade, order, reason + assert freqtrade.handle_cancel_exit(trade, order, reason ) == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] # Message should not be iterated again assert trade.sell_order_status == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] assert send_msg_mock.call_count == 1 -def test_handle_cancel_sell_cancel_exception(mocker, default_conf) -> None: +def test_handle_cancel_exit_cancel_exception(mocker, default_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch( @@ -2630,7 +2630,7 @@ def test_handle_cancel_sell_cancel_exception(mocker, default_conf) -> None: order = {'remaining': 1, 'amount': 1, 'status': "open"} - assert freqtrade.handle_cancel_sell(trade, order, reason) == 'error cancelling order' + assert freqtrade.handle_cancel_exit(trade, order, reason) == 'error cancelling order' def test_execute_trade_exit_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> None: @@ -4304,8 +4304,8 @@ def test_cancel_all_open_orders(mocker, default_conf, fee, limit_buy_order, limi mocker.patch('freqtrade.exchange.Exchange.fetch_order', side_effect=[ ExchangeError(), limit_sell_order, limit_buy_order, limit_sell_order]) - buy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_buy') - sell_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_sell') + buy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_enter') + sell_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_exit') freqtrade = get_patched_freqtradebot(mocker, default_conf) create_mock_trades(fee) From e1f846f22f28a9f76df1e8c47a79225ca2bc6e89 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 00:49:04 -0600 Subject: [PATCH 050/239] sell_lock -> exit_lock --- freqtrade/freqtradebot.py | 6 +++--- freqtrade/rpc/rpc.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index bf07ac0bc..0a0b20f0c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -99,7 +99,7 @@ class FreqtradeBot(LoggingMixin): self.state = State[initial_state.upper()] if initial_state else State.STOPPED # Protect sell-logic from forcesell and vice versa - self._sell_lock = Lock() + self._exit_lock = Lock() LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe)) def notify_status(self, msg: str) -> None: @@ -166,14 +166,14 @@ class FreqtradeBot(LoggingMixin): self.strategy.analyze(self.active_pair_whitelist) - with self._sell_lock: + with self._exit_lock: # Check and handle any timed out open orders self.check_handle_timedout() # Protect from collisions with forcesell. # Without this, freqtrade my try to recreate stoploss_on_exchange orders # while selling is in process, since telegram messages arrive in an different thread. - with self._sell_lock: + with self._exit_lock: trades = Trade.get_open_trades() # First process current opened trades (positions) self.exit_positions(trades) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index caf7345a2..b7b1fe603 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -566,7 +566,7 @@ class RPC: if self._freqtrade.state != State.RUNNING: raise RPCException('trader is not running') - with self._freqtrade._sell_lock: + with self._freqtrade._exit_lock: if trade_id == 'all': # Execute sell for all open orders for trade in Trade.get_open_trades(): @@ -628,7 +628,7 @@ class RPC: Handler for delete . Delete the given trade and close eventually existing open orders. """ - with self._freqtrade._sell_lock: + with self._freqtrade._exit_lock: c_count = 0 trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first() if not trade: From 362dc20406c0419246cbe459518636337567cf9b Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 00:45:55 -0600 Subject: [PATCH 051/239] notify_buy -> notify_enter, notify_sell -> notify_exit --- freqtrade/freqtradebot.py | 26 +++++++++++++------------- tests/test_freqtradebot.py | 6 +++--- tests/test_integration.py | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0a0b20f0c..b67ae9f00 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -590,11 +590,11 @@ class FreqtradeBot(LoggingMixin): # Updating wallets self.wallets.update() - self._notify_buy(trade, order_type) + self._notify_enter(trade, order_type) return True - def _notify_buy(self, trade: Trade, order_type: str) -> None: + def _notify_enter(self, trade: Trade, order_type: str) -> None: """ Sends rpc notification when a buy occurred. """ @@ -617,7 +617,7 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) - def _notify_buy_cancel(self, trade: Trade, order_type: str, reason: str) -> None: + def _notify_enter_cancel(self, trade: Trade, order_type: str, reason: str) -> None: """ Sends rpc notification when a buy cancel occurred. """ @@ -643,7 +643,7 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) - def _notify_buy_fill(self, trade: Trade) -> None: + def _notify_enter_fill(self, trade: Trade) -> None: msg = { 'trade_id': trade.id, 'type': RPCMessageType.BUY_FILL, @@ -782,7 +782,7 @@ class FreqtradeBot(LoggingMixin): # Lock pair for one candle to prevent immediate rebuys self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), reason='Auto lock') - self._notify_sell(trade, "stoploss") + self._notify_exit(trade, "stoploss") return True if trade.open_order_id or not trade.is_open: @@ -994,8 +994,8 @@ class FreqtradeBot(LoggingMixin): reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}" self.wallets.update() - self._notify_buy_cancel(trade, order_type=self.strategy.order_types['buy'], - reason=reason) + self._notify_enter_cancel(trade, order_type=self.strategy.order_types['buy'], + reason=reason) return was_trade_fully_canceled def handle_cancel_exit(self, trade: Trade, order: Dict, reason: str) -> str: @@ -1032,7 +1032,7 @@ class FreqtradeBot(LoggingMixin): reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] self.wallets.update() - self._notify_sell_cancel( + self._notify_exit_cancel( trade, order_type=self.strategy.order_types['sell'], reason=reason @@ -1150,11 +1150,11 @@ class FreqtradeBot(LoggingMixin): self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), reason='Auto lock') - self._notify_sell(trade, order_type) + self._notify_exit(trade, order_type) return True - def _notify_sell(self, trade: Trade, order_type: str, fill: bool = False) -> None: + def _notify_exit(self, trade: Trade, order_type: str, fill: bool = False) -> None: """ Sends rpc notification when a sell occurred. """ @@ -1196,7 +1196,7 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) - def _notify_sell_cancel(self, trade: Trade, order_type: str, reason: str) -> None: + def _notify_exit_cancel(self, trade: Trade, order_type: str, reason: str) -> None: """ Sends rpc notification when a sell cancel occurred. """ @@ -1291,13 +1291,13 @@ class FreqtradeBot(LoggingMixin): # Updating wallets when order is closed if not trade.is_open: if not stoploss_order and not trade.open_order_id: - self._notify_sell(trade, '', True) + self._notify_exit(trade, '', True) self.protections.stop_per_pair(trade.pair) self.protections.global_stop() self.wallets.update() elif not trade.open_order_id: # Buy fill - self._notify_buy_fill(trade) + self._notify_enter_fill(trade) return False diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 1b5381264..26d06d53a 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2485,7 +2485,7 @@ def test_handle_cancel_enter(mocker, caplog, default_conf, limit_buy_order) -> N mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock) freqtrade = FreqtradeBot(default_conf) - freqtrade._notify_buy_cancel = MagicMock() + freqtrade._notify_enter_cancel = MagicMock() trade = MagicMock() trade.pair = 'LTC/USDT' @@ -2526,7 +2526,7 @@ def test_handle_cancel_enter_exchanges(mocker, caplog, default_conf, cancel_order_mock = mocker.patch( 'freqtrade.exchange.Exchange.cancel_order_with_result', return_value=limit_buy_order_canceled_empty) - nofiy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot._notify_buy_cancel') + nofiy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot._notify_enter_cancel') freqtrade = FreqtradeBot(default_conf) reason = CANCEL_REASON['TIMEOUT'] @@ -2555,7 +2555,7 @@ def test_handle_cancel_enter_corder_empty(mocker, default_conf, limit_buy_order, ) freqtrade = FreqtradeBot(default_conf) - freqtrade._notify_buy_cancel = MagicMock() + freqtrade._notify_enter_cancel = MagicMock() trade = MagicMock() trade.pair = 'LTC/USDT' diff --git a/tests/test_integration.py b/tests/test_integration.py index 215927098..a3484d438 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -70,7 +70,7 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee, mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', create_stoploss_order=MagicMock(return_value=True), - _notify_sell=MagicMock(), + _notify_exit=MagicMock(), ) mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock) wallets_mock = mocker.patch("freqtrade.wallets.Wallets.update", MagicMock()) @@ -154,7 +154,7 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, moc mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', create_stoploss_order=MagicMock(return_value=True), - _notify_sell=MagicMock(), + _notify_exit=MagicMock(), ) should_sell_mock = MagicMock(side_effect=[ SellCheckTuple(sell_type=SellType.NONE), From a1c9a4d619f3dfe5f123e928487f5dcdf1d351fc Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 01:40:22 -0600 Subject: [PATCH 052/239] freqtradebot local name changes --- freqtrade/freqtradebot.py | 48 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index b67ae9f00..5800befba 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -476,21 +476,21 @@ class FreqtradeBot(LoggingMixin): time_in_force = self.strategy.order_time_in_force['buy'] if price: - buy_limit_requested = price + enter_limit_requested = price else: # Calculate price - proposed_buy_rate = self.exchange.get_rate(pair, refresh=True, side="buy") + proposed_enter_rate = self.exchange.get_rate(pair, refresh=True, side="buy") custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price, - default_retval=proposed_buy_rate)( + default_retval=proposed_enter_rate)( pair=pair, current_time=datetime.now(timezone.utc), - proposed_rate=proposed_buy_rate) + proposed_rate=proposed_enter_rate) - buy_limit_requested = self.get_valid_price(custom_entry_price, proposed_buy_rate) + enter_limit_requested = self.get_valid_price(custom_entry_price, proposed_enter_rate) - if not buy_limit_requested: + if not enter_limit_requested: raise PricingError('Could not determine buy price.') - min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, buy_limit_requested, + min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, enter_limit_requested, self.strategy.stoploss) if not self.edge: @@ -498,7 +498,7 @@ class FreqtradeBot(LoggingMixin): stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount, default_retval=stake_amount)( pair=pair, current_time=datetime.now(timezone.utc), - current_rate=buy_limit_requested, proposed_stake=stake_amount, + current_rate=enter_limit_requested, proposed_stake=stake_amount, min_stake=min_stake_amount, max_stake=max_stake_amount) stake_amount = self.wallets._validate_stake_amount(pair, stake_amount, min_stake_amount) @@ -508,27 +508,27 @@ class FreqtradeBot(LoggingMixin): logger.info(f"Buy signal found: about create a new trade for {pair} with stake_amount: " f"{stake_amount} ...") - amount = stake_amount / buy_limit_requested + amount = stake_amount / enter_limit_requested order_type = self.strategy.order_types['buy'] if forcebuy: # Forcebuy can define a different ordertype order_type = self.strategy.order_types.get('forcebuy', order_type) if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( - pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested, + pair=pair, order_type=order_type, amount=amount, rate=enter_limit_requested, time_in_force=time_in_force, current_time=datetime.now(timezone.utc)): logger.info(f"User requested abortion of buying {pair}") return False amount = self.exchange.amount_to_precision(pair, amount) order = self.exchange.create_order(pair=pair, ordertype=order_type, side="buy", - amount=amount, rate=buy_limit_requested, + amount=amount, rate=enter_limit_requested, time_in_force=time_in_force) order_obj = Order.parse_from_ccxt_object(order, pair, 'buy') order_id = order['id'] order_status = order.get('status', None) # we assume the order is executed at the price requested - buy_limit_filled_price = buy_limit_requested + enter_limit_filled_price = enter_limit_requested amount_requested = amount if order_status == 'expired' or order_status == 'rejected': @@ -551,13 +551,13 @@ class FreqtradeBot(LoggingMixin): ) stake_amount = order['cost'] amount = safe_value_fallback(order, 'filled', 'amount') - buy_limit_filled_price = safe_value_fallback(order, 'average', 'price') + enter_limit_filled_price = safe_value_fallback(order, 'average', 'price') # in case of FOK the order may be filled immediately and fully elif order_status == 'closed': stake_amount = order['cost'] amount = safe_value_fallback(order, 'filled', 'amount') - buy_limit_filled_price = safe_value_fallback(order, 'average', 'price') + enter_limit_filled_price = safe_value_fallback(order, 'average', 'price') # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') @@ -569,8 +569,8 @@ class FreqtradeBot(LoggingMixin): amount_requested=amount_requested, fee_open=fee, fee_close=fee, - open_rate=buy_limit_filled_price, - open_rate_requested=buy_limit_requested, + open_rate=enter_limit_filled_price, + open_rate_requested=enter_limit_requested, open_date=datetime.utcnow(), exchange=self.exchange.id, open_order_id=order_id, @@ -713,8 +713,8 @@ class FreqtradeBot(LoggingMixin): ) logger.debug('checking sell') - sell_rate = self.exchange.get_rate(trade.pair, refresh=True, side="sell") - if self._check_and_execute_sell(trade, sell_rate, buy, sell): + exit_rate = self.exchange.get_rate(trade.pair, refresh=True, side="sell") + if self._check_and_execute_exit(trade, exit_rate, buy, sell): return True logger.debug('Found no sell signal for %s.', trade) @@ -744,7 +744,7 @@ class FreqtradeBot(LoggingMixin): except InvalidOrderException as e: trade.stoploss_order_id = None logger.error(f'Unable to place a stoploss order on exchange. {e}') - logger.warning('Selling the trade forcefully') + logger.warning('Exiting the trade forcefully') self.execute_trade_exit(trade, trade.stop_loss, sell_reason=SellCheckTuple( sell_type=SellType.EMERGENCY_SELL)) @@ -851,19 +851,19 @@ class FreqtradeBot(LoggingMixin): logger.warning(f"Could not create trailing stoploss order " f"for pair {trade.pair}.") - def _check_and_execute_sell(self, trade: Trade, sell_rate: float, - buy: bool, sell: bool) -> bool: + def _check_and_execute_exit(self, trade: Trade, exit_rate: float, + enter: bool, exit_: bool) -> bool: """ - Check and execute sell + Check and execute exit """ should_sell = self.strategy.should_sell( - trade, sell_rate, datetime.now(timezone.utc), buy, sell, + trade, exit_rate, datetime.now(timezone.utc), buy, sell, force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0 ) if should_sell.sell_flag: logger.info(f'Executing Sell for {trade.pair}. Reason: {should_sell.sell_type}') - self.execute_trade_exit(trade, sell_rate, should_sell) + self.execute_trade_exit(trade, exit_rate, should_sell) return True return False From b2f289e4040ae676830987c3436ba85f8b29667d Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 02:16:25 -0600 Subject: [PATCH 053/239] Fixed freqtradebot failing tests --- freqtrade/freqtradebot.py | 2 +- tests/test_freqtradebot.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5800befba..7f668273c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -852,7 +852,7 @@ class FreqtradeBot(LoggingMixin): f"for pair {trade.pair}.") def _check_and_execute_exit(self, trade: Trade, exit_rate: float, - enter: bool, exit_: bool) -> bool: + buy: bool, sell: bool) -> bool: """ Check and execute exit """ diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 26d06d53a..3c5a8cfae 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1190,7 +1190,7 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, assert trade.stoploss_order_id is None assert trade.sell_reason == SellType.EMERGENCY_SELL.value assert log_has("Unable to place a stoploss order on exchange. ", caplog) - assert log_has("Selling the trade forcefully", caplog) + assert log_has("Exiting the trade forcefully", caplog) # Should call a market sell assert create_order_mock.call_count == 2 From 366247dff3bdd360b5b5fdc43c36e1396c902d91 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Thu, 9 Sep 2021 02:16:24 -0600 Subject: [PATCH 054/239] removed caplog.clears at end of functions in test_freqtradebot --- tests/test_freqtradebot.py | 45 -------------------------------------- 1 file changed, 45 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index ddd031f77..a9760978e 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -107,7 +107,6 @@ def test_order_dict_dry_run(default_conf, mocker, caplog) -> None: freqtrade = FreqtradeBot(conf) assert not freqtrade.strategy.order_types['stoploss_on_exchange'] assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) - caplog.clear() def test_order_dict_live(default_conf, mocker, caplog) -> None: @@ -141,7 +140,6 @@ def test_order_dict_live(default_conf, mocker, caplog) -> None: freqtrade = FreqtradeBot(conf) assert not freqtrade.strategy.order_types['stoploss_on_exchange'] assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) - caplog.clear() def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None: @@ -417,7 +415,6 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord assert freqtrade.create_trade('ETH/BTC') assert log_has_re(r"Stake amount for pair .* is too small.*", caplog) - caplog.clear() def test_create_trade_zero_stake_amount(default_conf, ticker, limit_buy_order_open, @@ -481,7 +478,6 @@ def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_ope n = freqtrade.enter_positions() assert n == 0 assert log_has_re(r"No currency pair in active pair whitelist.*", caplog) - caplog.clear() def test_enter_positions_no_pairs_in_whitelist(default_conf, ticker, limit_buy_order, fee, @@ -528,7 +524,6 @@ def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order, n = freqtrade.enter_positions() assert n == 0 assert log_has_re(message, caplog) - caplog.clear() def test_create_trade_no_signal(default_conf, fee, mocker) -> None: @@ -1122,7 +1117,6 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) assert freqtrade.handle_stoploss_on_exchange(trade) is False assert stoploss.call_count == 0 - caplog.clear() def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, @@ -1162,7 +1156,6 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, assert log_has_re(r'Stoploss order was cancelled, but unable to recreate one.*', caplog) assert trade.stoploss_order_id is None assert trade.is_open is True - caplog.clear() def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, @@ -1211,7 +1204,6 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, assert rpc_mock.call_count == 2 assert rpc_mock.call_args_list[1][0][0]['sell_reason'] == SellType.EMERGENCY_SELL.value assert rpc_mock.call_args_list[1][0][0]['order_type'] == 'market' - caplog.clear() def test_create_stoploss_order_insufficient_funds(mocker, default_conf, caplog, fee, @@ -1438,7 +1430,6 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging) assert cancel_mock.call_count == 1 assert log_has_re(r"Could not create trailing stoploss order for pair ETH/BTC\..*", caplog) - caplog.clear() @pytest.mark.usefixtures("init_persistence") @@ -1673,7 +1664,6 @@ def test_enter_positions(mocker, default_conf, caplog) -> None: assert log_has('Found no buy signals for whitelisted currencies. Trying again...', caplog) # create_trade should be called once for every pair in the whitelist. assert mock_ct.call_count == len(default_conf['exchange']['pair_whitelist']) - caplog.clear() def test_enter_positions_exception(mocker, default_conf, caplog) -> None: @@ -1713,7 +1703,6 @@ def test_exit_positions(mocker, default_conf, limit_buy_order, caplog) -> None: # test amount modified by fee-logic n = freqtrade.exit_positions(trades) assert n == 0 - caplog.clear() def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog) -> None: @@ -1734,7 +1723,6 @@ def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog) n = freqtrade.exit_positions(trades) assert n == 0 assert log_has('Unable to sell trade ETH/BTC: ', caplog) - caplog.clear() def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> None: @@ -1780,7 +1768,6 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No freqtrade.update_trade_state(trade, '123') assert log_has_re('Found open order for.*', caplog) - caplog.clear() def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_buy_order, fee, @@ -1831,7 +1818,6 @@ def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_ assert trade.amount != amount assert trade.amount == limit_buy_order['amount'] assert log_has_re(r'Applying fee on amount for .*', caplog) - caplog.clear() def test_update_trade_state_exception(mocker, default_conf, @@ -1850,7 +1836,6 @@ def test_update_trade_state_exception(mocker, default_conf, ) freqtrade.update_trade_state(trade, trade.open_order_id) assert log_has('Could not update trade amount: ', caplog) - caplog.clear() def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None: @@ -1867,7 +1852,6 @@ def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None freqtrade.update_trade_state(trade, trade.open_order_id) assert grm_mock.call_count == 0 assert log_has(f'Unable to fetch order {trade.open_order_id}: ', caplog) - caplog.clear() def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_order_open, @@ -2036,7 +2020,6 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order_open, assert freqtrade.handle_trade(trade) assert log_has("ETH/BTC - Required profit reached. sell_type=SellType.ROI", caplog) - caplog.clear() def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_open, @@ -2069,7 +2052,6 @@ def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_open assert freqtrade.handle_trade(trade) assert log_has("ETH/BTC - Sell signal received. sell_type=SellType.SELL_SIGNAL", caplog) - caplog.clear() def test_close_trade(default_conf, ticker, limit_buy_order, limit_buy_order_open, limit_sell_order, @@ -2110,7 +2092,6 @@ def test_bot_loop_start_called_once(mocker, default_conf, caplog): assert log_has_re(r'Strategy caused the following exception.*', caplog) assert ftbot.strategy.bot_loop_start.call_count == 1 assert ftbot.strategy.analyze.call_count == 1 - caplog.clear() def test_check_handle_timedout_buy_usercustom(default_conf, ticker, limit_buy_order_old, open_trade, @@ -2225,7 +2206,6 @@ def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, o nb_trades = len(trades) assert nb_trades == 0 assert log_has_re("Buy order cancelled on exchange for Trade.*", caplog) - caplog.clear() def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_order_old, open_trade, @@ -2360,7 +2340,6 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old, assert rpc_mock.call_count == 1 assert open_trade.is_open is True assert log_has_re("Sell order cancelled on exchange for Trade.*", caplog) - caplog.clear() def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, @@ -2429,7 +2408,6 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap assert trades[0].open_order_id is None assert trades[0].fee_updated('buy') assert pytest.approx(trades[0].fee_open) == 0.001 - caplog.clear() def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, caplog, fee, @@ -2470,7 +2448,6 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, limit_buy_order_old_partial['remaining']) assert trades[0].open_order_id is None assert trades[0].fee_open == fee() - caplog.clear() def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocker, caplog) -> None: @@ -2499,7 +2476,6 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke f"{open_trade.open_date.strftime('%Y-%m-%d %H:%M:%S')}" r"\) due to Traceback \(most recent call last\):\n*", caplog) - caplog.clear() def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> None: @@ -2543,7 +2519,6 @@ def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> Non mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock) assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) assert log_has_re(r"Order .* for .* not cancelled.", caplog) - caplog.clear() @pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'ftx', 'kraken', 'bittrex'], @@ -2565,7 +2540,6 @@ def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf, assert cancel_order_mock.call_count == 0 assert log_has_re(r'Buy order fully cancelled. Removing .* from database\.', caplog) assert nofiy_mock.call_count == 1 - caplog.clear() @pytest.mark.parametrize('cancelorder', [ @@ -2935,7 +2909,6 @@ def test_execute_trade_exit_sloe_cancel_exception( sell_reason=SellCheckTuple(sell_type=SellType.STOP_LOSS)) assert create_order_mock.call_count == 2 assert log_has('Could not cancel stoploss order abcd', caplog) - caplog.clear() def test_execute_trade_exit_with_stoploss_on_exchange(default_conf, ticker, fee, ticker_sell_up, @@ -3331,7 +3304,6 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_ assert freqtrade.handle_trade(trade) is True assert log_has_re(r'.*Falling back to wallet-amount.', caplog) assert trade.amount != amnt - caplog.clear() def test__safe_sell_amount(default_conf, fee, caplog, mocker): @@ -3362,7 +3334,6 @@ def test__safe_sell_amount(default_conf, fee, caplog, mocker): assert freqtrade._safe_sell_amount(trade.pair, amount_wallet) == amount_wallet assert not log_has_re(r'.*Falling back to wallet-amount.', caplog) assert wallet_update.call_count == 1 - caplog.clear() def test__safe_sell_amount_error(default_conf, fee, caplog, mocker): @@ -3419,7 +3390,6 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo freqtrade.enter_positions() assert log_has_re(f"Pair {trade.pair} is still locked.*", caplog) - caplog.clear() def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, @@ -3511,7 +3481,6 @@ def test_trailing_stop_loss(default_conf, limit_buy_order_open, limit_buy_order, assert log_has("ETH/BTC - HIT STOP: current price at 0.000012, stoploss is 0.000015, " "initial stoploss was at 0.000010, trade opened at 0.000011", caplog) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value - caplog.clear() def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_order_open, fee, @@ -3573,7 +3542,6 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_or f"ETH/BTC - HIT STOP: current price at {buy_price + 0.000002:.6f}, " f"stoploss is {trade.stop_loss:.6f}, " f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog) - caplog.clear() def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_order_open, fee, @@ -3636,7 +3604,6 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_orde f"stoploss is {trade.stop_loss:.6f}, " f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value - caplog.clear() def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_open, fee, @@ -3701,7 +3668,6 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_ assert log_has("ETH/BTC - Using positive stoploss: 0.05 offset: 0.055 profit: 0.1218%", caplog) assert log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000117705 - caplog.clear() def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, @@ -3763,7 +3729,6 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).', caplog) - caplog.clear() def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fee, fee, @@ -3789,7 +3754,6 @@ def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fe assert walletmock.call_count == 1 assert log_has_re(r'Fee amount for Trade.* was in base currency ' '- Eating Fee 0.008 into dust', caplog) - caplog.clear() def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, fee): @@ -3812,7 +3776,6 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' 'open_rate=0.24544100, open_since=closed) failed: myTrade-Dict empty found', caplog) - caplog.clear() def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fee, mocker): @@ -3906,7 +3869,6 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c assert trade.fee_open_currency is not None assert trade.fee_close_cost is None assert trade.fee_close_currency is None - caplog.clear() def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, caplog, fee, @@ -3942,7 +3904,6 @@ def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, assert trade.fee_open_currency is not None assert trade.fee_close_cost is None assert trade.fee_close_currency is None - caplog.clear() def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, fee, @@ -3971,7 +3932,6 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).', caplog) - caplog.clear() def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, fee, mocker): @@ -4215,7 +4175,6 @@ def test_order_book_bid_strategy_exception(mocker, default_conf, caplog) -> None with pytest.raises(PricingError): freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") assert log_has_re(r'Buy Price at location 1 from orderbook could not be determined.', caplog) - caplog.clear() def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: @@ -4286,7 +4245,6 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order_open, limit_buy_o freqtrade.handle_trade(trade) assert log_has_re(r'Sell Price at location 1 from orderbook could not be determined\..*', caplog) - caplog.clear() def test_startup_state(default_conf, mocker): @@ -4345,7 +4303,6 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order_ assert log_has_re(r"Unable to create trade for XRP/BTC: " r"Available balance \(0.0 BTC\) is lower than stake amount \(0.001 BTC\)", caplog) - caplog.clear() @pytest.mark.usefixtures("init_persistence") @@ -4505,7 +4462,6 @@ def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): assert log_has_re(r"Trying to reupdate buy fees for .*", caplog) assert mock_uts.call_count == 0 assert not log_has_re(r"Updating buy-fee on trade .* for order .*\.", caplog) - caplog.clear() @pytest.mark.usefixtures("init_persistence") @@ -4643,7 +4599,6 @@ def test_refind_lost_order(mocker, default_conf, fee, caplog): freqtrade.refind_lost_order(trades[4]) assert log_has(f"Error updating {order['id']}.", caplog) - caplog.clear() def test_get_valid_price(mocker, default_conf) -> None: From 982534ddc70b7db43c598d7a48a2ff63c81aaedf Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Sep 2021 20:56:46 +0200 Subject: [PATCH 055/239] Add gate.io to list of supported exchanges --- README.md | 1 + docs/index.md | 1 + freqtrade/exchange/gateio.py | 2 ++ 3 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 309fab94b..2c164135f 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even - [X] [Bittrex](https://bittrex.com/) - [X] [Kraken](https://kraken.com/) - [X] [FTX](https://ftx.com) +- [X] [Gate.io](https://www.gate.io/ref/6266643) - [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ### Community tested diff --git a/docs/index.md b/docs/index.md index fd3b8f224..7735117e2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -40,6 +40,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual, - [X] [Bittrex](https://bittrex.com/) - [X] [FTX](https://ftx.com) - [X] [Kraken](https://kraken.com/) +- [X] [Gate.io](https://www.gate.io/ref/6266643) - [ ] [potentially many others through ccxt](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ### Community tested diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index 9c910a10d..e6ee01c8a 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -21,3 +21,5 @@ class Gateio(Exchange): _ft_has: Dict = { "ohlcv_candle_limit": 1000, } + + _headers = {'X-Gate-Channel-Id': 'freqtrade'} From a19c33ba54ff051dfa20aae864510ac2a8bb6626 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Sep 2021 05:52:48 +0000 Subject: [PATCH 056/239] Don't blindly create coroutines, but fire them off in batches --- freqtrade/exchange/exchange.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 79fd33dfe..48c3caa50 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -28,7 +28,7 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier, retrier_async) -from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 +from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -1238,19 +1238,20 @@ class Exchange: input_coroutines = [self._async_get_candle_history( pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] - - results = await asyncio.gather(*input_coroutines, return_exceptions=True) - # Combine gathered results + data: List = [] - for res in results: - if isinstance(res, Exception): - logger.warning("Async code raised an exception: %s", res.__class__.__name__) - continue - # Deconstruct tuple if it's not an exception - p, _, new_data = res - if p == pair: - data.extend(new_data) + for input_coro in chunks(input_coroutines, 100): + + results = await asyncio.gather(*input_coro, return_exceptions=True) + for res in results: + if isinstance(res, Exception): + logger.warning("Async code raised an exception: %s", res.__class__.__name__) + continue + # Deconstruct tuple if it's not an exception + p, _, new_data = res + if p == pair: + data.extend(new_data) # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) logger.info("Downloaded data for %s with length %s.", pair, len(data)) From 8c9159f5969f593aa36754af9f19e4667223ba0a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Sep 2021 19:46:38 +0200 Subject: [PATCH 057/239] Improve comments --- freqtrade/exchange/exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 48c3caa50..e79d53e80 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1238,9 +1238,9 @@ class Exchange: input_coroutines = [self._async_get_candle_history( pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] - # Combine gathered results data: List = [] + # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling for input_coro in chunks(input_coroutines, 100): results = await asyncio.gather(*input_coro, return_exceptions=True) From 4c4604f837d05d1d3c391031f24e8b63cef2da48 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Sep 2021 19:47:56 +0200 Subject: [PATCH 058/239] Add explicit test for get_historic_ohlcv --- tests/exchange/test_exchange.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 144063c07..45c2e627c 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1551,6 +1551,32 @@ def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name): assert 'high' in ret.columns +@pytest.mark.asyncio +@pytest.mark.parametrize("exchange_name", EXCHANGES) +async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): + ohlcv = [ + [ + int((datetime.now(timezone.utc).timestamp() - 1000) * 1000), + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) + + pair = 'ETH/BTC' + res = await exchange._async_get_historic_ohlcv(pair, "5m", + 1500000000000, is_new_pair=False) + # Call with very old timestamp - causes tons of requests + assert exchange._api_async.fetch_ohlcv.call_count > 200 + assert res[0] == ohlcv[0] + assert log_has_re(r'Downloaded data for .* with length .*\.', caplog) + + def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: ohlcv = [ [ From a5f90a409cb5c78373bf2c477ac5dc62958cbf70 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Sep 2021 08:18:32 +0200 Subject: [PATCH 059/239] Small updates to async_history_fetch --- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_exchange.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index e79d53e80..11de5411f 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1254,7 +1254,7 @@ class Exchange: data.extend(new_data) # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) - logger.info("Downloaded data for %s with length %s.", pair, len(data)) + logger.info(f"Downloaded data for {pair} with length {len(data)}.") return data def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 45c2e627c..90e19782e 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1568,7 +1568,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ # Monkey-patch async function exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) - pair = 'ETH/BTC' + pair = 'ETH/USDT' res = await exchange._async_get_historic_ohlcv(pair, "5m", 1500000000000, is_new_pair=False) # Call with very old timestamp - causes tons of requests From 432c3df17e7672ab33554d161a4f7fb3e1f218ac Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Sep 2021 08:04:22 +0200 Subject: [PATCH 060/239] Add documentation for Bittex/Gemini with VolumePairlist closes #5565 --- docs/exchanges.md | 6 ++++++ docs/includes/pairlists.md | 20 ++++++++++++++++++++ freqtrade/plugins/pairlist/VolumePairList.py | 2 +- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index 42a850acd..c0fbdc694 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -58,6 +58,12 @@ Bittrex does not support market orders. If you have a message at the bot startup Bittrex also does not support `VolumePairlist` due to limited / split API constellation at the moment. Please use `StaticPairlist`. Other pairlists (other than `VolumePairlist`) should not be affected. +### Volume pairlist + +Bittrex does not support the direct usage of VolumePairList. This can however be worked around by using the advanced mode with `lookback_days: 1` (or more), which will emulate 24h volume. + +Read more in the [pairlist documentation](plugins.md#volumepairlist-advanced-mode). + ### Restricted markets Bittrex split its exchange into US and International versions. diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 6e23c9003..69e12d5dc 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -82,6 +82,8 @@ Filtering instances (not the first position in the list) will not apply any cach You can define a minimum volume with `min_value` - which will filter out pairs with a volume lower than the specified value in the specified timerange. +### VolumePairList Advanced mode + `VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles. For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days: @@ -105,6 +107,24 @@ For convenience `lookback_days` can be specified, which will imply that 1d candl !!! Warning "Performance implications when using lookback range" If used in first position in combination with lookback, the computation of the range based volume can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation. +??? Tip "Unsupported exchanges (Bittrex, Gemini)" + On some exchanges (like Bittrex and Gemini), regular VolumePairList does not work as the api does not natively provide 24h volume. This can be worked around by using candle data to build the volume. + To roughly simulate 24h volume, you can use the following configuration. + Please note that These pairlists will only refresh once per day. + + ```json + "pairlists": [ + { + "method": "VolumePairList", + "number_assets": 20, + "sort_key": "quoteVolume", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 1 + } + ], + ``` + More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles: ```json diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index c70e4a904..0ffc8a8c8 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -123,7 +123,7 @@ class VolumePairList(IPairList): filtered_tickers = [ v for k, v in tickers.items() if (self._exchange.get_pair_quote_currency(k) == self._stake_currency - and v[self._sort_key] is not None)] + and (self._use_range or v[self._sort_key] is not None))] pairlist = [s['symbol'] for s in filtered_tickers] pairlist = self.filter_pairlist(pairlist, tickers) From ebb0b8aa3fd03100d96e8feef4a4f9919fab60fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Sep 2021 08:36:57 +0200 Subject: [PATCH 061/239] Remove new-hyperopt command --- docs/utils.md | 35 ----- freqtrade/commands/__init__.py | 2 +- freqtrade/commands/arguments.py | 17 +-- freqtrade/commands/deploy_commands.py | 52 +------ freqtrade/templates/base_hyperopt.py.j2 | 137 ------------------ .../subtemplates/hyperopt_buy_guards_full.j2 | 8 - .../hyperopt_buy_guards_minimal.j2 | 2 - .../subtemplates/hyperopt_buy_space_full.j2 | 9 -- .../hyperopt_buy_space_minimal.j2 | 3 - .../subtemplates/hyperopt_sell_guards_full.j2 | 8 - .../hyperopt_sell_guards_minimal.j2 | 2 - .../subtemplates/hyperopt_sell_space_full.j2 | 11 -- .../hyperopt_sell_space_minimal.j2 | 5 - tests/commands/test_commands.py | 33 +---- 14 files changed, 7 insertions(+), 317 deletions(-) delete mode 100644 freqtrade/templates/base_hyperopt.py.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_buy_guards_full.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_buy_guards_minimal.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_buy_space_full.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_buy_space_minimal.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_sell_guards_full.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_sell_guards_minimal.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_sell_space_full.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_sell_space_minimal.j2 diff --git a/docs/utils.md b/docs/utils.md index 6395fb6f9..a1e219d6a 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -111,41 +111,6 @@ Using the advanced template (populates all optional functions and methods) freqtrade new-strategy --strategy AwesomeStrategy --template advanced ``` -## Create new hyperopt - -Creates a new hyperopt from a template similar to SampleHyperopt. -The file will be named inline with your class name, and will not overwrite existing files. - -Results will be located in `user_data/hyperopts/.py`. - -``` output -usage: freqtrade new-hyperopt [-h] [--userdir PATH] [--hyperopt NAME] - [--template {full,minimal,advanced}] - -optional arguments: - -h, --help show this help message and exit - --userdir PATH, --user-data-dir PATH - Path to userdata directory. - --hyperopt NAME Specify hyperopt class name which will be used by the - bot. - --template {full,minimal,advanced} - Use a template which is either `minimal`, `full` - (containing multiple sample indicators) or `advanced`. - Default: `full`. -``` - -### Sample usage of new-hyperopt - -```bash -freqtrade new-hyperopt --hyperopt AwesomeHyperopt -``` - -With custom user directory - -```bash -freqtrade new-hyperopt --userdir ~/.freqtrade/ --hyperopt AwesomeHyperopt -``` - ## List Strategies and List Hyperopts Use the `list-strategies` subcommand to see all strategies in one particular directory and the `list-hyperopts` subcommand to list custom Hyperopts. diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 04e46ee23..49b184a80 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -11,7 +11,7 @@ from freqtrade.commands.build_config_commands import start_new_config from freqtrade.commands.data_commands import (start_convert_data, start_download_data, start_list_data) from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui, - start_new_hyperopt, start_new_strategy) + start_new_strategy) from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show from freqtrade.commands.list_commands import (start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 899998310..9c6ce543f 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -55,8 +55,6 @@ ARGS_BUILD_CONFIG = ["config"] ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] -ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] - ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] @@ -95,7 +93,7 @@ NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list "list-hyperopts", "hyperopt-list", "hyperopt-show", "plot-dataframe", "plot-profit", "show-trades"] -NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"] +NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] class Arguments: @@ -176,10 +174,9 @@ class Arguments: start_hyperopt_list, start_hyperopt_show, start_install_ui, start_list_data, start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, - start_list_timeframes, start_new_config, start_new_hyperopt, - start_new_strategy, start_plot_dataframe, start_plot_profit, - start_show_trades, start_test_pairlist, start_trading, - start_webserver) + start_list_timeframes, start_new_config, start_new_strategy, + start_plot_dataframe, start_plot_profit, start_show_trades, + start_test_pairlist, start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -206,12 +203,6 @@ class Arguments: build_config_cmd.set_defaults(func=start_new_config) self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd) - # add new-hyperopt subcommand - build_hyperopt_cmd = subparsers.add_parser('new-hyperopt', - help="Create new hyperopt") - build_hyperopt_cmd.set_defaults(func=start_new_hyperopt) - self._build_args(optionlist=ARGS_BUILD_HYPEROPT, parser=build_hyperopt_cmd) - # add new-strategy subcommand build_strategy_cmd = subparsers.add_parser('new-strategy', help="Create new strategy") diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index c98335e0b..4f9e5bbad 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -7,7 +7,7 @@ import requests from freqtrade.configuration import setup_utils_configuration from freqtrade.configuration.directory_operations import copy_sample_files, create_userdata_dir -from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES +from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.misc import render_template, render_template_with_fallback @@ -87,56 +87,6 @@ def start_new_strategy(args: Dict[str, Any]) -> None: raise OperationalException("`new-strategy` requires --strategy to be set.") -def deploy_new_hyperopt(hyperopt_name: str, hyperopt_path: Path, subtemplate: str) -> None: - """ - Deploys a new hyperopt template to hyperopt_path - """ - fallback = 'full' - buy_guards = render_template_with_fallback( - templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/hyperopt_buy_guards_{fallback}.j2", - ) - sell_guards = render_template_with_fallback( - templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/hyperopt_sell_guards_{fallback}.j2", - ) - buy_space = render_template_with_fallback( - templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/hyperopt_buy_space_{fallback}.j2", - ) - sell_space = render_template_with_fallback( - templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/hyperopt_sell_space_{fallback}.j2", - ) - - strategy_text = render_template(templatefile='base_hyperopt.py.j2', - arguments={"hyperopt": hyperopt_name, - "buy_guards": buy_guards, - "sell_guards": sell_guards, - "buy_space": buy_space, - "sell_space": sell_space, - }) - - logger.info(f"Writing hyperopt to `{hyperopt_path}`.") - hyperopt_path.write_text(strategy_text) - - -def start_new_hyperopt(args: Dict[str, Any]) -> None: - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - if 'hyperopt' in args and args['hyperopt']: - - new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args['hyperopt'] + '.py') - - if new_path.exists(): - raise OperationalException(f"`{new_path}` already exists. " - "Please choose another Hyperopt Name.") - deploy_new_hyperopt(args['hyperopt'], new_path, args['template']) - else: - raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") - - def clean_ui_subdir(directory: Path): if directory.is_dir(): logger.info("Removing UI directory content.") diff --git a/freqtrade/templates/base_hyperopt.py.j2 b/freqtrade/templates/base_hyperopt.py.j2 deleted file mode 100644 index f6ca1477a..000000000 --- a/freqtrade/templates/base_hyperopt.py.j2 +++ /dev/null @@ -1,137 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement - -# --- Do not remove these libs --- -from functools import reduce -from typing import Any, Callable, Dict, List - -import numpy as np # noqa -import pandas as pd # noqa -from pandas import DataFrame -from skopt.space import Categorical, Dimension, Integer, Real # noqa - -from freqtrade.optimize.hyperopt_interface import IHyperOpt - -# -------------------------------- -# Add your lib to import here -import talib.abstract as ta # noqa -import freqtrade.vendor.qtpylib.indicators as qtpylib - - -class {{ hyperopt }}(IHyperOpt): - """ - This is a Hyperopt template to get you started. - - More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/ - - You should: - - Add any lib you need to build your hyperopt. - - You must keep: - - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator. - - The methods roi_space, generate_roi_table and stoploss_space are not required - and are provided by default. - However, you may override them if you need 'roi' and 'stoploss' spaces that - differ from the defaults offered by Freqtrade. - Sample implementation of these methods will be copied to `user_data/hyperopts` when - creating the user-data directory using `freqtrade create-userdir --userdir user_data`, - or is available online under the following URL: - https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py. - """ - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching buy strategy parameters. - """ - return [ - {{ buy_space | indent(12) }} - ] - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - {{ buy_guards | indent(12) }} - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - # Check that the candle had volume - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters. - """ - return [ - {{ sell_space | indent(12) }} - ] - - @staticmethod - def sell_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the sell strategy parameters to be used by Hyperopt. - """ - def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Sell strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - {{ sell_guards | indent(12) }} - - # TRIGGERS - if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_upper': - conditions.append(dataframe['close'] > dataframe['bb_upperband']) - if params['sell-trigger'] == 'sell-macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) - if params['sell-trigger'] == 'sell-sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['sar'], dataframe['close'] - )) - - # Check that the candle had volume - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 - - return dataframe - - return populate_sell_trend - diff --git a/freqtrade/templates/subtemplates/hyperopt_buy_guards_full.j2 b/freqtrade/templates/subtemplates/hyperopt_buy_guards_full.j2 deleted file mode 100644 index 5b967f4ed..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_buy_guards_full.j2 +++ /dev/null @@ -1,8 +0,0 @@ -if params.get('mfi-enabled'): - conditions.append(dataframe['mfi'] < params['mfi-value']) -if params.get('fastd-enabled'): - conditions.append(dataframe['fastd'] < params['fastd-value']) -if params.get('adx-enabled'): - conditions.append(dataframe['adx'] > params['adx-value']) -if params.get('rsi-enabled'): - conditions.append(dataframe['rsi'] < params['rsi-value']) diff --git a/freqtrade/templates/subtemplates/hyperopt_buy_guards_minimal.j2 b/freqtrade/templates/subtemplates/hyperopt_buy_guards_minimal.j2 deleted file mode 100644 index 5e1022f59..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_buy_guards_minimal.j2 +++ /dev/null @@ -1,2 +0,0 @@ -if params.get('rsi-enabled'): - conditions.append(dataframe['rsi'] < params['rsi-value']) diff --git a/freqtrade/templates/subtemplates/hyperopt_buy_space_full.j2 b/freqtrade/templates/subtemplates/hyperopt_buy_space_full.j2 deleted file mode 100644 index 29bafbd93..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_buy_space_full.j2 +++ /dev/null @@ -1,9 +0,0 @@ -Integer(10, 25, name='mfi-value'), -Integer(15, 45, name='fastd-value'), -Integer(20, 50, name='adx-value'), -Integer(20, 40, name='rsi-value'), -Categorical([True, False], name='mfi-enabled'), -Categorical([True, False], name='fastd-enabled'), -Categorical([True, False], name='adx-enabled'), -Categorical([True, False], name='rsi-enabled'), -Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') diff --git a/freqtrade/templates/subtemplates/hyperopt_buy_space_minimal.j2 b/freqtrade/templates/subtemplates/hyperopt_buy_space_minimal.j2 deleted file mode 100644 index 5ddf537fb..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_buy_space_minimal.j2 +++ /dev/null @@ -1,3 +0,0 @@ -Integer(20, 40, name='rsi-value'), -Categorical([True, False], name='rsi-enabled'), -Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') diff --git a/freqtrade/templates/subtemplates/hyperopt_sell_guards_full.j2 b/freqtrade/templates/subtemplates/hyperopt_sell_guards_full.j2 deleted file mode 100644 index bd7b499f4..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_sell_guards_full.j2 +++ /dev/null @@ -1,8 +0,0 @@ -if params.get('sell-mfi-enabled'): - conditions.append(dataframe['mfi'] > params['sell-mfi-value']) -if params.get('sell-fastd-enabled'): - conditions.append(dataframe['fastd'] > params['sell-fastd-value']) -if params.get('sell-adx-enabled'): - conditions.append(dataframe['adx'] < params['sell-adx-value']) -if params.get('sell-rsi-enabled'): - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) diff --git a/freqtrade/templates/subtemplates/hyperopt_sell_guards_minimal.j2 b/freqtrade/templates/subtemplates/hyperopt_sell_guards_minimal.j2 deleted file mode 100644 index 8b4adebf6..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_sell_guards_minimal.j2 +++ /dev/null @@ -1,2 +0,0 @@ -if params.get('sell-rsi-enabled'): - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) diff --git a/freqtrade/templates/subtemplates/hyperopt_sell_space_full.j2 b/freqtrade/templates/subtemplates/hyperopt_sell_space_full.j2 deleted file mode 100644 index 46469d532..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_sell_space_full.j2 +++ /dev/null @@ -1,11 +0,0 @@ -Integer(75, 100, name='sell-mfi-value'), -Integer(50, 100, name='sell-fastd-value'), -Integer(50, 100, name='sell-adx-value'), -Integer(60, 100, name='sell-rsi-value'), -Categorical([True, False], name='sell-mfi-enabled'), -Categorical([True, False], name='sell-fastd-enabled'), -Categorical([True, False], name='sell-adx-enabled'), -Categorical([True, False], name='sell-rsi-enabled'), -Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') diff --git a/freqtrade/templates/subtemplates/hyperopt_sell_space_minimal.j2 b/freqtrade/templates/subtemplates/hyperopt_sell_space_minimal.j2 deleted file mode 100644 index dfb110543..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_sell_space_minimal.j2 +++ /dev/null @@ -1,5 +0,0 @@ -Integer(60, 100, name='sell-rsi-value'), -Categorical([True, False], name='sell-rsi-enabled'), -Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 1da9e5100..8db2020b5 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -12,7 +12,7 @@ from freqtrade.commands import (start_convert_data, start_create_userdir, start_ start_hyperopt_list, start_hyperopt_show, start_install_ui, start_list_data, start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, start_list_timeframes, - start_new_hyperopt, start_new_strategy, start_show_trades, + start_new_strategy, start_show_trades, start_test_pairlist, start_trading, start_webserver) from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) @@ -519,37 +519,6 @@ def test_start_new_strategy_no_arg(mocker, caplog): start_new_strategy(get_args(args)) -def test_start_new_hyperopt(mocker, caplog): - wt_mock = mocker.patch.object(Path, "write_text", MagicMock()) - mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - - args = [ - "new-hyperopt", - "--hyperopt", - "CoolNewhyperopt" - ] - start_new_hyperopt(get_args(args)) - - assert wt_mock.call_count == 1 - assert "CoolNewhyperopt" in wt_mock.call_args_list[0][0][0] - assert log_has_re("Writing hyperopt to .*", caplog) - - mocker.patch('freqtrade.commands.deploy_commands.setup_utils_configuration') - mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - with pytest.raises(OperationalException, - match=r".* already exists. Please choose another Hyperopt Name\."): - start_new_hyperopt(get_args(args)) - - -def test_start_new_hyperopt_no_arg(mocker): - args = [ - "new-hyperopt", - ] - with pytest.raises(OperationalException, - match="`new-hyperopt` requires --hyperopt to be set."): - start_new_hyperopt(get_args(args)) - - def test_start_install_ui(mocker): clean_mock = mocker.patch('freqtrade.commands.deploy_commands.clean_ui_subdir') get_url_mock = mocker.patch('freqtrade.commands.deploy_commands.get_ui_download_url', From dad4a49e81c3a01eb98166b6db20fb08bb1a3a9a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Sep 2021 09:06:57 +0200 Subject: [PATCH 062/239] Remove legacy hyperopt interface from hyperopt.py --- freqtrade/commands/cli_options.py | 2 +- freqtrade/optimize/hyperopt.py | 64 ++--- freqtrade/optimize/hyperopt_auto.py | 21 +- freqtrade/optimize/hyperopt_interface.py | 14 +- freqtrade/resolvers/hyperopt_resolver.py | 38 --- freqtrade/templates/sample_hyperopt.py | 174 ----------- .../templates/sample_hyperopt_advanced.py | 269 ------------------ 7 files changed, 25 insertions(+), 557 deletions(-) delete mode 100644 freqtrade/templates/sample_hyperopt.py delete mode 100644 freqtrade/templates/sample_hyperopt_advanced.py diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index cf7cb804c..a1790cb9a 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -209,7 +209,7 @@ AVAILABLE_CLI_OPTIONS = { ), "hyperopt_path": Arg( '--hyperopt-path', - help='Specify additional lookup path for Hyperopt and Hyperopt Loss functions.', + help='Specify additional lookup path for Hyperopt Loss functions.', metavar='PATH', ), "epochs": Arg( diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e0b35df32..d37c68769 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,6 +22,7 @@ from pandas import DataFrame from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN from freqtrade.data.converter import trim_dataframes from freqtrade.data.history import get_timerange +from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, file_dump_json, plural from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules @@ -30,7 +31,7 @@ from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401 from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer from freqtrade.optimize.optimize_reports import generate_strategy_stats -from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver +from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver # Suppress scikit-learn FutureWarnings from skopt @@ -80,8 +81,9 @@ class Hyperopt: self.custom_hyperopt = HyperOptAuto(self.config) self.auto_hyperopt = True else: - self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config) - self.auto_hyperopt = False + raise OperationalException( + "Using seperate Hyperopt files has been removed in 2021.9. Please convert " + "your existing Hyperopt file to the new Hyperoptable strategy interface") self.backtesting._set_strategy(self.backtesting.strategylist[0]) self.custom_hyperopt.strategy = self.backtesting.strategy @@ -103,31 +105,6 @@ class Hyperopt: self.num_epochs_saved = 0 self.current_best_epoch: Optional[Dict[str, Any]] = None - if not self.auto_hyperopt: - # Populate "fallback" functions here - # (hasattr is slow so should not be run during "regular" operations) - if hasattr(self.custom_hyperopt, 'populate_indicators'): - logger.warning( - "DEPRECATED: Using `populate_indicators()` in the hyperopt file is deprecated. " - "Please move these methods to your strategy." - ) - self.backtesting.strategy.populate_indicators = ( # type: ignore - self.custom_hyperopt.populate_indicators) # type: ignore - if hasattr(self.custom_hyperopt, 'populate_buy_trend'): - logger.warning( - "DEPRECATED: Using `populate_buy_trend()` in the hyperopt file is deprecated. " - "Please move these methods to your strategy." - ) - self.backtesting.strategy.populate_buy_trend = ( # type: ignore - self.custom_hyperopt.populate_buy_trend) # type: ignore - if hasattr(self.custom_hyperopt, 'populate_sell_trend'): - logger.warning( - "DEPRECATED: Using `populate_sell_trend()` in the hyperopt file is deprecated. " - "Please move these methods to your strategy." - ) - self.backtesting.strategy.populate_sell_trend = ( # type: ignore - self.custom_hyperopt.populate_sell_trend) # type: ignore - # Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): self.max_open_trades = self.config['max_open_trades'] @@ -256,7 +233,7 @@ class Hyperopt: """ Assign the dimensions in the hyperoptimization space. """ - if self.auto_hyperopt and HyperoptTools.has_space(self.config, 'protection'): + if HyperoptTools.has_space(self.config, 'protection'): # Protections can only be optimized when using the Parameter interface logger.debug("Hyperopt has 'protection' space") # Enable Protections if protection space is selected. @@ -285,6 +262,15 @@ class Hyperopt: self.dimensions = (self.buy_space + self.sell_space + self.protection_space + self.roi_space + self.stoploss_space + self.trailing_space) + def assign_params(self, params_dict: Dict, category: str) -> None: + """ + Assign hyperoptable parameters + """ + for attr_name, attr in self.backtesting.strategy.enumerate_parameters(category): + if attr.optimize: + # noinspection PyProtectedMember + attr.value = params_dict[attr_name] + def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict: """ Used Optimize function. @@ -296,18 +282,13 @@ class Hyperopt: # Apply parameters if HyperoptTools.has_space(self.config, 'buy'): - self.backtesting.strategy.advise_buy = ( # type: ignore - self.custom_hyperopt.buy_strategy_generator(params_dict)) + self.assign_params(params_dict, 'buy') if HyperoptTools.has_space(self.config, 'sell'): - self.backtesting.strategy.advise_sell = ( # type: ignore - self.custom_hyperopt.sell_strategy_generator(params_dict)) + self.assign_params(params_dict, 'sell') if HyperoptTools.has_space(self.config, 'protection'): - for attr_name, attr in self.backtesting.strategy.enumerate_parameters('protection'): - if attr.optimize: - # noinspection PyProtectedMember - attr.value = params_dict[attr_name] + self.assign_params(params_dict, 'protection') if HyperoptTools.has_space(self.config, 'roi'): self.backtesting.strategy.minimal_roi = ( # type: ignore @@ -517,11 +498,10 @@ class Hyperopt: f"saved to '{self.results_file}'.") if self.current_best_epoch: - if self.auto_hyperopt: - HyperoptTools.try_export_params( - self.config, - self.backtesting.strategy.get_strategy_name(), - self.current_best_epoch) + HyperoptTools.try_export_params( + self.config, + self.backtesting.strategy.get_strategy_name(), + self.current_best_epoch) HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs, self.print_json) diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py index 43e92d9c6..022f04a84 100644 --- a/freqtrade/optimize/hyperopt_auto.py +++ b/freqtrade/optimize/hyperopt_auto.py @@ -22,26 +22,6 @@ class HyperOptAuto(IHyperOpt): sell_indicator_space methods, but other hyperopt methods can be overridden as well. """ - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: - def populate_buy_trend(dataframe: DataFrame, metadata: dict): - for attr_name, attr in self.strategy.enumerate_parameters('buy'): - if attr.optimize: - # noinspection PyProtectedMember - attr.value = params[attr_name] - return self.strategy.populate_buy_trend(dataframe, metadata) - - return populate_buy_trend - - def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable: - def populate_sell_trend(dataframe: DataFrame, metadata: dict): - for attr_name, attr in self.strategy.enumerate_parameters('sell'): - if attr.optimize: - # noinspection PyProtectedMember - attr.value = params[attr_name] - return self.strategy.populate_sell_trend(dataframe, metadata) - - return populate_sell_trend - def _get_func(self, name) -> Callable: """ Return a function defined in Strategy.HyperOpt class, or one defined in super() class. @@ -61,6 +41,7 @@ class HyperOptAuto(IHyperOpt): yield attr.get_space(attr_name) def _get_indicator_space(self, category, fallback_method_name): + # TODO: is this necessary, or can we call "generate_space" directly? indicator_space = list(self._generate_indicator_space(category)) if len(indicator_space) > 0: return indicator_space diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 500798627..814260f5e 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -5,7 +5,7 @@ This module defines the interface to apply for hyperopt import logging import math from abc import ABC -from typing import Any, Callable, Dict, List +from typing import Dict, List from skopt.space import Categorical, Dimension, Integer @@ -45,18 +45,6 @@ class IHyperOpt(ABC): IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED IHyperOpt.timeframe = str(config['timeframe']) - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: - """ - Create a buy strategy generator. - """ - raise OperationalException(_format_exception_message('buy_strategy_generator', 'buy')) - - def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable: - """ - Create a sell strategy generator. - """ - raise OperationalException(_format_exception_message('sell_strategy_generator', 'sell')) - def protection_space(self) -> List[Dimension]: """ Create a protection space. diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index 8327a4d13..6f0263e93 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -9,7 +9,6 @@ from typing import Dict from freqtrade.constants import HYPEROPT_LOSS_BUILTIN, USERPATH_HYPEROPTS from freqtrade.exceptions import OperationalException -from freqtrade.optimize.hyperopt_interface import IHyperOpt from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss from freqtrade.resolvers import IResolver @@ -17,43 +16,6 @@ from freqtrade.resolvers import IResolver logger = logging.getLogger(__name__) -class HyperOptResolver(IResolver): - """ - This class contains all the logic to load custom hyperopt class - """ - object_type = IHyperOpt - object_type_str = "Hyperopt" - user_subdir = USERPATH_HYPEROPTS - initial_search_path = None - - @staticmethod - def load_hyperopt(config: Dict) -> IHyperOpt: - """ - Load the custom hyperopt class from config parameter - :param config: configuration dictionary - """ - if not config.get('hyperopt'): - raise OperationalException("No Hyperopt set. Please use `--hyperopt` to specify " - "the Hyperopt class to use.") - - hyperopt_name = config['hyperopt'] - - hyperopt = HyperOptResolver.load_object(hyperopt_name, config, - kwargs={'config': config}, - extra_dir=config.get('hyperopt_path')) - - if not hasattr(hyperopt, 'populate_indicators'): - logger.info("Hyperopt class does not provide populate_indicators() method. " - "Using populate_indicators from the strategy.") - if not hasattr(hyperopt, 'populate_buy_trend'): - logger.info("Hyperopt class does not provide populate_buy_trend() method. " - "Using populate_buy_trend from the strategy.") - if not hasattr(hyperopt, 'populate_sell_trend'): - logger.info("Hyperopt class does not provide populate_sell_trend() method. " - "Using populate_sell_trend from the strategy.") - return hyperopt - - class HyperOptLossResolver(IResolver): """ This class contains all the logic to load custom hyperopt loss class diff --git a/freqtrade/templates/sample_hyperopt.py b/freqtrade/templates/sample_hyperopt.py deleted file mode 100644 index ed1af7718..000000000 --- a/freqtrade/templates/sample_hyperopt.py +++ /dev/null @@ -1,174 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement -# isort: skip_file - -# --- Do not remove these libs --- -from functools import reduce -from typing import Any, Callable, Dict, List - -import numpy as np # noqa -import pandas as pd # noqa -from pandas import DataFrame -from skopt.space import Categorical, Dimension, Integer, Real # noqa - -from freqtrade.optimize.hyperopt_interface import IHyperOpt - -# -------------------------------- -# Add your lib to import here -import talib.abstract as ta # noqa -import freqtrade.vendor.qtpylib.indicators as qtpylib - - -class SampleHyperOpt(IHyperOpt): - """ - This is a sample Hyperopt to inspire you. - - More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/ - - You should: - - Rename the class name to some unique name. - - Add any methods you want to build your hyperopt. - - Add any lib you need to build your hyperopt. - - An easier way to get a new hyperopt file is by using - `freqtrade new-hyperopt --hyperopt MyCoolHyperopt`. - - You must keep: - - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator. - - The methods roi_space, generate_roi_table and stoploss_space are not required - and are provided by default. - However, you may override them if you need 'roi' and 'stoploss' spaces that - differ from the defaults offered by Freqtrade. - Sample implementation of these methods will be copied to `user_data/hyperopts` when - creating the user-data directory using `freqtrade create-userdir --userdir user_data`, - or is available online under the following URL: - https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py. - """ - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching buy strategy parameters. - """ - return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') - ] - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - if 'mfi-enabled' in params and params['mfi-enabled']: - conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd-enabled' in params and params['fastd-enabled']: - conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters. - """ - return [ - Integer(75, 100, name='sell-mfi-value'), - Integer(50, 100, name='sell-fastd-value'), - Integer(50, 100, name='sell-adx-value'), - Integer(60, 100, name='sell-rsi-value'), - Categorical([True, False], name='sell-mfi-enabled'), - Categorical([True, False], name='sell-fastd-enabled'), - Categorical([True, False], name='sell-adx-enabled'), - Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') - ] - - @staticmethod - def sell_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the sell strategy parameters to be used by Hyperopt. - """ - def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Sell strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: - conditions.append(dataframe['mfi'] > params['sell-mfi-value']) - if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: - conditions.append(dataframe['fastd'] > params['sell-fastd-value']) - if 'sell-adx-enabled' in params and params['sell-adx-enabled']: - conditions.append(dataframe['adx'] < params['sell-adx-value']) - if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) - - # TRIGGERS - if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_upper': - conditions.append(dataframe['close'] > dataframe['bb_upperband']) - if params['sell-trigger'] == 'sell-macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) - if params['sell-trigger'] == 'sell-sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['sar'], dataframe['close'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 - - return dataframe - - return populate_sell_trend diff --git a/freqtrade/templates/sample_hyperopt_advanced.py b/freqtrade/templates/sample_hyperopt_advanced.py deleted file mode 100644 index cc13b6ba3..000000000 --- a/freqtrade/templates/sample_hyperopt_advanced.py +++ /dev/null @@ -1,269 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement -# isort: skip_file -# --- Do not remove these libs --- -from functools import reduce -from typing import Any, Callable, Dict, List - -import numpy as np # noqa -import pandas as pd # noqa -from pandas import DataFrame -from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal, Real # noqa - -from freqtrade.optimize.hyperopt_interface import IHyperOpt - -# -------------------------------- -# Add your lib to import here -import talib.abstract as ta # noqa -import freqtrade.vendor.qtpylib.indicators as qtpylib - - -class AdvancedSampleHyperOpt(IHyperOpt): - """ - This is a sample hyperopt to inspire you. - Feel free to customize it. - - More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/ - - You should: - - Rename the class name to some unique name. - - Add any methods you want to build your hyperopt. - - Add any lib you need to build your hyperopt. - - You must keep: - - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator. - - The methods roi_space, generate_roi_table and stoploss_space are not required - and are provided by default. - However, you may override them if you need the - 'roi' and the 'stoploss' spaces that differ from the defaults offered by Freqtrade. - - This sample illustrates how to override these methods. - """ - @staticmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - This method can also be loaded from the strategy, if it doesn't exist in the hyperopt class. - """ - dataframe['adx'] = ta.ADX(dataframe) - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['mfi'] = ta.MFI(dataframe) - dataframe['rsi'] = ta.RSI(dataframe) - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_upperband'] = bollinger['upper'] - dataframe['sar'] = ta.SAR(dataframe) - return dataframe - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching buy strategy parameters. - """ - return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') - ] - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by hyperopt - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use - """ - conditions = [] - # GUARDS AND TRENDS - if 'mfi-enabled' in params and params['mfi-enabled']: - conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd-enabled' in params and params['fastd-enabled']: - conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters. - """ - return [ - Integer(75, 100, name='sell-mfi-value'), - Integer(50, 100, name='sell-fastd-value'), - Integer(50, 100, name='sell-adx-value'), - Integer(60, 100, name='sell-rsi-value'), - Categorical([True, False], name='sell-mfi-enabled'), - Categorical([True, False], name='sell-fastd-enabled'), - Categorical([True, False], name='sell-adx-enabled'), - Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') - ] - - @staticmethod - def sell_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the sell strategy parameters to be used by hyperopt - """ - def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Sell strategy Hyperopt will build and use - """ - # print(params) - conditions = [] - # GUARDS AND TRENDS - if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: - conditions.append(dataframe['mfi'] > params['sell-mfi-value']) - if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: - conditions.append(dataframe['fastd'] > params['sell-fastd-value']) - if 'sell-adx-enabled' in params and params['sell-adx-enabled']: - conditions.append(dataframe['adx'] < params['sell-adx-value']) - if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) - - # TRIGGERS - if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_upper': - conditions.append(dataframe['close'] > dataframe['bb_upperband']) - if params['sell-trigger'] == 'sell-macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) - if params['sell-trigger'] == 'sell-sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['sar'], dataframe['close'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 - - return dataframe - - return populate_sell_trend - - @staticmethod - def generate_roi_table(params: Dict) -> Dict[int, float]: - """ - Generate the ROI table that will be used by Hyperopt - - This implementation generates the default legacy Freqtrade ROI tables. - - Change it if you need different number of steps in the generated - ROI tables or other structure of the ROI tables. - - Please keep it aligned with parameters in the 'roi' optimization - hyperspace defined by the roi_space method. - """ - roi_table = {} - roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] - roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] - roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 - - return roi_table - - @staticmethod - def roi_space() -> List[Dimension]: - """ - Values to search for each ROI steps - - Override it if you need some different ranges for the parameters in the - 'roi' optimization hyperspace. - - Please keep it aligned with the implementation of the - generate_roi_table method. - """ - return [ - Integer(10, 120, name='roi_t1'), - Integer(10, 60, name='roi_t2'), - Integer(10, 40, name='roi_t3'), - SKDecimal(0.01, 0.04, decimals=3, name='roi_p1'), - SKDecimal(0.01, 0.07, decimals=3, name='roi_p2'), - SKDecimal(0.01, 0.20, decimals=3, name='roi_p3'), - ] - - @staticmethod - def stoploss_space() -> List[Dimension]: - """ - Stoploss Value to search - - Override it if you need some different range for the parameter in the - 'stoploss' optimization hyperspace. - """ - return [ - SKDecimal(-0.35, -0.02, decimals=3, name='stoploss'), - ] - - @staticmethod - def trailing_space() -> List[Dimension]: - """ - Create a trailing stoploss space. - - You may override it in your custom Hyperopt class. - """ - return [ - # It was decided to always set trailing_stop is to True if the 'trailing' hyperspace - # is used. Otherwise hyperopt will vary other parameters that won't have effect if - # trailing_stop is set False. - # This parameter is included into the hyperspace dimensions rather than assigning - # it explicitly in the code in order to have it printed in the results along with - # other 'trailing' hyperspace parameters. - Categorical([True], name='trailing_stop'), - - SKDecimal(0.01, 0.35, decimals=3, name='trailing_stop_positive'), - - # 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive', - # so this intermediate parameter is used as the value of the difference between - # them. The value of the 'trailing_stop_positive_offset' is constructed in the - # generate_trailing_params() method. - # This is similar to the hyperspace dimensions used for constructing the ROI tables. - SKDecimal(0.001, 0.1, decimals=3, name='trailing_stop_positive_offset_p1'), - - Categorical([True, False], name='trailing_only_offset_is_reached'), - ] From fd6bf591f8abb888e3ad340bcc9c368096ae92c9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Sep 2021 17:11:02 +0200 Subject: [PATCH 063/239] Update some tests to remove explicit hyperopt interface --- freqtrade/optimize/hyperopt.py | 3 +- freqtrade/optimize/hyperopt_auto.py | 22 ++- freqtrade/optimize/hyperopt_interface.py | 27 --- tests/commands/test_commands.py | 43 ++--- tests/optimize/conftest.py | 2 +- tests/optimize/test_hyperopt.py | 203 ++++------------------- 6 files changed, 63 insertions(+), 237 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index d37c68769..14b155546 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -79,10 +79,9 @@ class Hyperopt: if not self.config.get('hyperopt'): self.custom_hyperopt = HyperOptAuto(self.config) - self.auto_hyperopt = True else: raise OperationalException( - "Using seperate Hyperopt files has been removed in 2021.9. Please convert " + "Using separate Hyperopt files has been removed in 2021.9. Please convert " "your existing Hyperopt file to the new Hyperoptable strategy interface") self.backtesting._set_strategy(self.backtesting.strategylist[0]) diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py index 022f04a84..1f11cec80 100644 --- a/freqtrade/optimize/hyperopt_auto.py +++ b/freqtrade/optimize/hyperopt_auto.py @@ -4,9 +4,9 @@ This module implements a convenience auto-hyperopt class, which can be used toge that implement IHyperStrategy interface. """ from contextlib import suppress -from typing import Any, Callable, Dict, List +from typing import Callable, Dict, List -from pandas import DataFrame +from freqtrade.exceptions import OperationalException with suppress(ImportError): @@ -15,6 +15,14 @@ with suppress(ImportError): from freqtrade.optimize.hyperopt_interface import IHyperOpt +def _format_exception_message(space: str) -> str: + raise OperationalException( + f"The '{space}' space is included into the hyperoptimization " + f"but no parameter for this space was not found in your Strategy. " + f"Please make sure to have parameters for this space enabled for optimization " + f"or remove the '{space}' space from hyperoptimization.") + + class HyperOptAuto(IHyperOpt): """ This class delegates functionality to Strategy(IHyperStrategy) and Strategy.HyperOpt classes. @@ -40,22 +48,22 @@ class HyperOptAuto(IHyperOpt): if attr.optimize: yield attr.get_space(attr_name) - def _get_indicator_space(self, category, fallback_method_name): + def _get_indicator_space(self, category): # TODO: is this necessary, or can we call "generate_space" directly? indicator_space = list(self._generate_indicator_space(category)) if len(indicator_space) > 0: return indicator_space else: - return self._get_func(fallback_method_name)() + _format_exception_message(category) def indicator_space(self) -> List['Dimension']: - return self._get_indicator_space('buy', 'indicator_space') + return self._get_indicator_space('buy') def sell_indicator_space(self) -> List['Dimension']: - return self._get_indicator_space('sell', 'sell_indicator_space') + return self._get_indicator_space('sell') def protection_space(self) -> List['Dimension']: - return self._get_indicator_space('protection', 'protection_space') + return self._get_indicator_space('protection') def generate_roi_table(self, params: Dict) -> Dict[int, float]: return self._get_func('generate_roi_table')(params) diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 814260f5e..8fb40f557 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -9,7 +9,6 @@ from typing import Dict, List from skopt.space import Categorical, Dimension, Integer -from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import round_dict from freqtrade.optimize.space import SKDecimal @@ -19,13 +18,6 @@ from freqtrade.strategy import IStrategy logger = logging.getLogger(__name__) -def _format_exception_message(method: str, space: str) -> str: - return (f"The '{space}' space is included into the hyperoptimization " - f"but {method}() method is not found in your " - f"custom Hyperopt class. You should either implement this " - f"method or remove the '{space}' space from hyperoptimization.") - - class IHyperOpt(ABC): """ Interface for freqtrade hyperopt @@ -45,25 +37,6 @@ class IHyperOpt(ABC): IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED IHyperOpt.timeframe = str(config['timeframe']) - def protection_space(self) -> List[Dimension]: - """ - Create a protection space. - Only supported by the Parameter interface. - """ - raise OperationalException(_format_exception_message('indicator_space', 'protection')) - - def indicator_space(self) -> List[Dimension]: - """ - Create an indicator space. - """ - raise OperationalException(_format_exception_message('indicator_space', 'buy')) - - def sell_indicator_space(self) -> List[Dimension]: - """ - Create a sell indicator space. - """ - raise OperationalException(_format_exception_message('sell_indicator_space', 'sell')) - def generate_roi_table(self, params: Dict) -> Dict[int, float]: """ Create a ROI table. diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 8db2020b5..a6a87d245 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -10,10 +10,10 @@ import pytest from freqtrade.commands import (start_convert_data, start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, start_install_ui, - start_list_data, start_list_exchanges, start_list_hyperopts, - start_list_markets, start_list_strategies, start_list_timeframes, - start_new_strategy, start_show_trades, - start_test_pairlist, start_trading, start_webserver) + start_list_data, start_list_exchanges, start_list_markets, + start_list_strategies, start_list_timeframes, start_new_strategy, + start_show_trades, start_test_pairlist, start_trading, + start_webserver) from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) from freqtrade.configuration import setup_utils_configuration @@ -793,37 +793,20 @@ def test_start_list_strategies(mocker, caplog, capsys): assert "legacy_strategy_v1.py" in captured.out assert "StrategyTestV2" in captured.out - -def test_start_list_hyperopts(mocker, caplog, capsys): - + # Test color output args = [ - "list-hyperopts", - "--hyperopt-path", - str(Path(__file__).parent.parent / "optimize" / "hyperopts"), - "-1" + "list-strategies", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy" / "strats"), ] pargs = get_args(args) # pargs['config'] = None - start_list_hyperopts(pargs) + start_list_strategies(pargs) captured = capsys.readouterr() - assert "TestHyperoptLegacy" not in captured.out - assert "legacy_hyperopt.py" not in captured.out - assert "HyperoptTestSepFile" in captured.out - assert "test_hyperopt.py" not in captured.out - - # Test regular output - args = [ - "list-hyperopts", - "--hyperopt-path", - str(Path(__file__).parent.parent / "optimize" / "hyperopts"), - ] - pargs = get_args(args) - # pargs['config'] = None - start_list_hyperopts(pargs) - captured = capsys.readouterr() - assert "TestHyperoptLegacy" not in captured.out - assert "legacy_hyperopt.py" not in captured.out - assert "HyperoptTestSepFile" in captured.out + assert "TestStrategyLegacyV1" in captured.out + assert "legacy_strategy_v1.py" in captured.out + assert "StrategyTestV2" in captured.out + assert "LOAD FAILED" in captured.out def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): diff --git a/tests/optimize/conftest.py b/tests/optimize/conftest.py index 95c9fef97..5c5171c3a 100644 --- a/tests/optimize/conftest.py +++ b/tests/optimize/conftest.py @@ -16,7 +16,7 @@ def hyperopt_conf(default_conf): hyperconf.update({ 'datadir': Path(default_conf['datadir']), 'runmode': RunMode.HYPEROPT, - 'hyperopt': 'HyperoptTestSepFile', + 'strategy': 'HyperoptableStrategy', 'hyperopt_loss': 'ShortTradeDurHyperOptLoss', 'hyperopt_path': str(Path(__file__).parent / 'hyperopts'), 'epochs': 1, diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 565d6077a..498b6e588 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -17,7 +17,6 @@ from freqtrade.optimize.hyperopt_auto import HyperOptAuto from freqtrade.optimize.hyperopt_tools import HyperoptTools from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.optimize.space import SKDecimal -from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver from freqtrade.strategy.hyper import IntParameter from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -133,47 +132,6 @@ def test_setup_hyperopt_configuration_stake_amount(mocker, default_conf) -> None setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) -def test_hyperoptresolver(mocker, default_conf, caplog) -> None: - patched_configuration_load_config_file(mocker, default_conf) - - hyperopt = HyperoptTestSepFile - delattr(hyperopt, 'populate_indicators') - delattr(hyperopt, 'populate_buy_trend') - delattr(hyperopt, 'populate_sell_trend') - mocker.patch( - 'freqtrade.resolvers.hyperopt_resolver.HyperOptResolver.load_object', - MagicMock(return_value=hyperopt(default_conf)) - ) - default_conf.update({'hyperopt': 'HyperoptTestSepFile'}) - x = HyperOptResolver.load_hyperopt(default_conf) - assert not hasattr(x, 'populate_indicators') - assert not hasattr(x, 'populate_buy_trend') - assert not hasattr(x, 'populate_sell_trend') - assert log_has("Hyperopt class does not provide populate_indicators() method. " - "Using populate_indicators from the strategy.", caplog) - assert log_has("Hyperopt class does not provide populate_sell_trend() method. " - "Using populate_sell_trend from the strategy.", caplog) - assert log_has("Hyperopt class does not provide populate_buy_trend() method. " - "Using populate_buy_trend from the strategy.", caplog) - assert hasattr(x, "ticker_interval") # DEPRECATED - assert hasattr(x, "timeframe") - - -def test_hyperoptresolver_wrongname(default_conf) -> None: - default_conf.update({'hyperopt': "NonExistingHyperoptClass"}) - - with pytest.raises(OperationalException, match=r'Impossible to load Hyperopt.*'): - HyperOptResolver.load_hyperopt(default_conf) - - -def test_hyperoptresolver_noname(default_conf): - default_conf['hyperopt'] = '' - with pytest.raises(OperationalException, - match="No Hyperopt set. Please use `--hyperopt` to specify " - "the Hyperopt class to use."): - HyperOptResolver.load_hyperopt(default_conf) - - def test_start_not_installed(mocker, default_conf, import_fails) -> None: start_mock = MagicMock() patched_configuration_load_config_file(mocker, default_conf) @@ -196,7 +154,7 @@ def test_start_not_installed(mocker, default_conf, import_fails) -> None: start_hyperopt(pargs) -def test_start(mocker, hyperopt_conf, caplog) -> None: +def test_start_no_hyperopt_allowed(mocker, hyperopt_conf, caplog) -> None: start_mock = MagicMock() patched_configuration_load_config_file(mocker, hyperopt_conf) mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) @@ -210,10 +168,8 @@ def test_start(mocker, hyperopt_conf, caplog) -> None: '--epochs', '5' ] pargs = get_args(args) - start_hyperopt(pargs) - - assert log_has('Starting freqtrade in Hyperopt mode', caplog) - assert start_mock.call_count == 1 + with pytest.raises(OperationalException, match=r"Using separate Hyperopt files has been.*"): + start_hyperopt(pargs) def test_start_no_data(mocker, hyperopt_conf) -> None: @@ -225,11 +181,11 @@ def test_start_no_data(mocker, hyperopt_conf) -> None: ) patch_exchange(mocker) - + # TODO: migrate to strategy-based hyperopt args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', '--hyperopt-loss', 'SharpeHyperOptLossDaily', '--epochs', '5' ] @@ -427,66 +383,14 @@ def test_hyperopt_format_results(hyperopt): def test_populate_indicators(hyperopt, testdatadir) -> None: data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) dataframes = hyperopt.backtesting.strategy.advise_all_indicators(data) - dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], - {'pair': 'UNITTEST/BTC'}) + dataframe = dataframes['UNITTEST/BTC'] # Check if some indicators are generated. We will not test all of them assert 'adx' in dataframe - assert 'mfi' in dataframe + assert 'macd' in dataframe assert 'rsi' in dataframe -def test_buy_strategy_generator(hyperopt, testdatadir) -> None: - data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) - dataframes = hyperopt.backtesting.strategy.advise_all_indicators(data) - dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], - {'pair': 'UNITTEST/BTC'}) - - populate_buy_trend = hyperopt.custom_hyperopt.buy_strategy_generator( - { - 'adx-value': 20, - 'fastd-value': 20, - 'mfi-value': 20, - 'rsi-value': 20, - 'adx-enabled': True, - 'fastd-enabled': True, - 'mfi-enabled': True, - 'rsi-enabled': True, - 'trigger': 'bb_lower' - } - ) - result = populate_buy_trend(dataframe, {'pair': 'UNITTEST/BTC'}) - # Check if some indicators are generated. We will not test all of them - assert 'buy' in result - assert 1 in result['buy'] - - -def test_sell_strategy_generator(hyperopt, testdatadir) -> None: - data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) - dataframes = hyperopt.backtesting.strategy.advise_all_indicators(data) - dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], - {'pair': 'UNITTEST/BTC'}) - - populate_sell_trend = hyperopt.custom_hyperopt.sell_strategy_generator( - { - 'sell-adx-value': 20, - 'sell-fastd-value': 75, - 'sell-mfi-value': 80, - 'sell-rsi-value': 20, - 'sell-adx-enabled': True, - 'sell-fastd-enabled': True, - 'sell-mfi-enabled': True, - 'sell-rsi-enabled': True, - 'sell-trigger': 'sell-bb_upper' - } - ) - result = populate_sell_trend(dataframe, {'pair': 'UNITTEST/BTC'}) - # Check if some indicators are generated. We will not test all of them - print(result) - assert 'sell' in result - assert 1 in result['sell'] - - def test_generate_optimizer(mocker, hyperopt_conf) -> None: hyperopt_conf.update({'spaces': 'all', 'hyperopt_min_trades': 1, @@ -527,24 +431,12 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: mocker.patch('freqtrade.optimize.hyperopt.load', return_value={'XRP/BTC': None}) optimizer_param = { - 'adx-value': 0, - 'fastd-value': 35, - 'mfi-value': 0, - 'rsi-value': 0, - 'adx-enabled': False, - 'fastd-enabled': True, - 'mfi-enabled': False, - 'rsi-enabled': False, - 'trigger': 'macd_cross_signal', - 'sell-adx-value': 0, - 'sell-fastd-value': 75, - 'sell-mfi-value': 0, - 'sell-rsi-value': 0, - 'sell-adx-enabled': False, - 'sell-fastd-enabled': True, - 'sell-mfi-enabled': False, - 'sell-rsi-enabled': False, - 'sell-trigger': 'macd_cross_signal', + 'buy_plusdi': 0.02, + 'buy_rsi': 35, + 'sell_minusdi': 0.02, + 'sell_rsi': 75, + 'protection_cooldown_lookback': 20, + 'protection_enabled': True, 'roi_t1': 60.0, 'roi_t2': 30.0, 'roi_t3': 20.0, @@ -564,29 +456,19 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: '0.00003100 BTC ( 0.00%). ' 'Avg duration 0:50:00 min.' ), - 'params_details': {'buy': {'adx-enabled': False, - 'adx-value': 0, - 'fastd-enabled': True, - 'fastd-value': 35, - 'mfi-enabled': False, - 'mfi-value': 0, - 'rsi-enabled': False, - 'rsi-value': 0, - 'trigger': 'macd_cross_signal'}, + 'params_details': {'buy': {'buy_plusdi': 0.02, + 'buy_rsi': 35, + }, 'roi': {"0": 0.12000000000000001, "20.0": 0.02, "50.0": 0.01, "110.0": 0}, - 'protection': {}, - 'sell': {'sell-adx-enabled': False, - 'sell-adx-value': 0, - 'sell-fastd-enabled': True, - 'sell-fastd-value': 75, - 'sell-mfi-enabled': False, - 'sell-mfi-value': 0, - 'sell-rsi-enabled': False, - 'sell-rsi-value': 0, - 'sell-trigger': 'macd_cross_signal'}, + 'protection': {'protection_cooldown_lookback': 20, + 'protection_enabled': True, + }, + 'sell': {'sell_minusdi': 0.02, + 'sell_rsi': 75, + }, 'stoploss': {'stoploss': -0.4}, 'trailing': {'trailing_only_offset_is_reached': False, 'trailing_stop': True, @@ -808,11 +690,6 @@ def test_simplified_interface_roi_stoploss(mocker, hyperopt_conf, capsys) -> Non hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - del hyperopt.custom_hyperopt.__class__.buy_strategy_generator - del hyperopt.custom_hyperopt.__class__.sell_strategy_generator - del hyperopt.custom_hyperopt.__class__.indicator_space - del hyperopt.custom_hyperopt.__class__.sell_indicator_space - hyperopt.start() parallel.assert_called_once() @@ -843,16 +720,14 @@ def test_simplified_interface_all_failed(mocker, hyperopt_conf) -> None: hyperopt_conf.update({'spaces': 'all', }) + mocker.patch('freqtrade.optimize.hyperopt_auto.HyperOptAuto._generate_indicator_space', + return_value=[]) + hyperopt = Hyperopt(hyperopt_conf) hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - del hyperopt.custom_hyperopt.__class__.buy_strategy_generator - del hyperopt.custom_hyperopt.__class__.sell_strategy_generator - del hyperopt.custom_hyperopt.__class__.indicator_space - del hyperopt.custom_hyperopt.__class__.sell_indicator_space - - with pytest.raises(OperationalException, match=r"The 'buy' space is included into *"): + with pytest.raises(OperationalException, match=r"The 'protection' space is included into *"): hyperopt.start() @@ -889,11 +764,6 @@ def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None: hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - # TODO: sell_strategy_generator() is actually not called because - # run_optimizer_parallel() is mocked - del hyperopt.custom_hyperopt.__class__.sell_strategy_generator - del hyperopt.custom_hyperopt.__class__.sell_indicator_space - hyperopt.start() parallel.assert_called_once() @@ -943,11 +813,6 @@ def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None: hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - # TODO: buy_strategy_generator() is actually not called because - # run_optimizer_parallel() is mocked - del hyperopt.custom_hyperopt.__class__.buy_strategy_generator - del hyperopt.custom_hyperopt.__class__.indicator_space - hyperopt.start() parallel.assert_called_once() @@ -964,13 +829,12 @@ def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None: assert hasattr(hyperopt, "position_stacking") -@pytest.mark.parametrize("method,space", [ - ('buy_strategy_generator', 'buy'), - ('indicator_space', 'buy'), - ('sell_strategy_generator', 'sell'), - ('sell_indicator_space', 'sell'), +@pytest.mark.parametrize("space", [ + ('buy'), + ('sell'), + ('protection'), ]) -def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> None: +def test_simplified_interface_failed(mocker, hyperopt_conf, space) -> None: mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', @@ -979,6 +843,8 @@ def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> No 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) + mocker.patch('freqtrade.optimize.hyperopt_auto.HyperOptAuto._generate_indicator_space', + return_value=[]) patch_exchange(mocker) @@ -988,8 +854,6 @@ def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> No hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - delattr(hyperopt.custom_hyperopt.__class__, method) - with pytest.raises(OperationalException, match=f"The '{space}' space is included into *"): hyperopt.start() @@ -999,7 +863,6 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) (Path(tmpdir) / 'hyperopt_results').mkdir(parents=True) # No hyperopt needed - del hyperopt_conf['hyperopt'] hyperopt_conf.update({ 'strategy': 'HyperoptableStrategy', 'user_data_dir': Path(tmpdir), From 3675df83448ceaaed1c92ec4c8b7d3273e1ef461 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Sep 2021 17:52:47 +0200 Subject: [PATCH 064/239] Update documentation regarding Legacy Hyperopt --- README.md | 8 +- docs/advanced-hyperopt.md | 298 +----------------- docs/bot-usage.md | 8 +- docs/deprecated.md | 5 + docs/faq.md | 2 +- docs/hyperopt.md | 21 +- docs/utils.md | 2 - freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 4 +- freqtrade/constants.py | 2 - tests/exchange/test_exchange.py | 4 +- .../hyperopts/hyperopt_test_sep_file.py | 202 ------------ tests/optimize/test_hyperopt.py | 14 +- tests/plugins/test_pairlocks.py | 2 +- tests/test_directory_operations.py | 8 +- 15 files changed, 55 insertions(+), 527 deletions(-) delete mode 100644 tests/optimize/hyperopts/hyperopt_test_sep_file.py diff --git a/README.md b/README.md index 2c164135f..01effd7bc 100644 --- a/README.md +++ b/README.md @@ -79,22 +79,22 @@ For any other type of installation please refer to [Installation doc](https://ww ``` usage: freqtrade [-h] [-V] - {trade,create-userdir,new-config,new-hyperopt,new-strategy,download-data,convert-data,convert-trade-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,plot-dataframe,plot-profit} + {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} ... Free, open source crypto trading bot positional arguments: - {trade,create-userdir,new-config,new-hyperopt,new-strategy,download-data,convert-data,convert-trade-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,plot-dataframe,plot-profit} + {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} trade Trade module. create-userdir Create user-data directory. new-config Create new config - new-hyperopt Create new hyperopt new-strategy Create new strategy download-data Download backtesting data. convert-data Convert candle (OHLCV) data from one format to another. convert-trade-data Convert trade data from one format to another. + list-data List downloaded data. backtesting Backtesting module. edge Edge module. hyperopt Hyperopt module. @@ -108,8 +108,10 @@ positional arguments: list-timeframes Print available timeframes for the exchange. show-trades Show trades. test-pairlist Test your pairlist configuration. + install-ui Install FreqUI plot-dataframe Plot candles with indicators. plot-profit Generate plot showing profits. + webserver Webserver module. optional arguments: -h, --help show this help message and exit diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 4a2bafd2e..f2f52b7dd 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -67,10 +67,10 @@ Currently, the arguments are: This function needs to return a floating point number (`float`). Smaller numbers will be interpreted as better results. The parameters and balancing for this is up to you. !!! Note - This function is called once per iteration - so please make sure to have this as optimized as possible to not slow hyperopt down unnecessarily. + This function is called once per epoch - so please make sure to have this as optimized as possible to not slow hyperopt down unnecessarily. -!!! Note - Please keep the arguments `*args` and `**kwargs` in the interface to allow us to extend this interface later. +!!! Note "`*args` and `**kwargs`" + Please keep the arguments `*args` and `**kwargs` in the interface to allow us to extend this interface in the future. ## Overriding pre-defined spaces @@ -82,8 +82,22 @@ class MyAwesomeStrategy(IStrategy): # Define a custom stoploss space. def stoploss_space(): return [SKDecimal(-0.05, -0.01, decimals=3, name='stoploss')] + + # Define custom ROI space + def roi_space() -> List[Dimension]: + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + SKDecimal(0.01, 0.04, decimals=3, name='roi_p1'), + SKDecimal(0.01, 0.07, decimals=3, name='roi_p2'), + SKDecimal(0.01, 0.20, decimals=3, name='roi_p3'), + ] ``` +!!! Note + All overrides are optional and can be mixed/matched as necessary. + ## Space options For the additional spaces, scikit-optimize (in combination with Freqtrade) provides the following space types: @@ -105,281 +119,3 @@ from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal, Assuming the definition of a rather small space (`SKDecimal(0.10, 0.15, decimals=2, name='xxx')`) - SKDecimal will have 5 possibilities (`[0.10, 0.11, 0.12, 0.13, 0.14, 0.15]`). A corresponding real space `Real(0.10, 0.15 name='xxx')` on the other hand has an almost unlimited number of possibilities (`[0.10, 0.010000000001, 0.010000000002, ... 0.014999999999, 0.01500000000]`). - ---- - -## Legacy Hyperopt - -This Section explains the configuration of an explicit Hyperopt file (separate to the strategy). - -!!! Warning "Deprecated / legacy mode" - Since the 2021.4 release you no longer have to write a separate hyperopt class, but all strategies can be hyperopted. - Please read the [main hyperopt page](hyperopt.md) for more details. - -### Prepare hyperopt file - -Configuring an explicit hyperopt file is similar to writing your own strategy, and many tasks will be similar. - -!!! Tip "About this page" - For this page, we will be using a fictional strategy called `AwesomeStrategy` - which will be optimized using the `AwesomeHyperopt` class. - -#### Create a Custom Hyperopt File - -The simplest way to get started is to use the following command, which will create a new hyperopt file from a template, which will be located under `user_data/hyperopts/AwesomeHyperopt.py`. - -Let assume you want a hyperopt file `AwesomeHyperopt.py`: - -``` bash -freqtrade new-hyperopt --hyperopt AwesomeHyperopt -``` - -#### Legacy Hyperopt checklist - -Checklist on all tasks / possibilities in hyperopt - -Depending on the space you want to optimize, only some of the below are required: - -* fill `buy_strategy_generator` - for buy signal optimization -* fill `indicator_space` - for buy signal optimization -* fill `sell_strategy_generator` - for sell signal optimization -* fill `sell_indicator_space` - for sell signal optimization - -!!! Note - `populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work. - -Optional in hyperopt - can also be loaded from a strategy (recommended): - -* `populate_indicators` - fallback to create indicators -* `populate_buy_trend` - fallback if not optimizing for buy space. should come from strategy -* `populate_sell_trend` - fallback if not optimizing for sell space. should come from strategy - -!!! Note - You always have to provide a strategy to Hyperopt, even if your custom Hyperopt class contains all methods. - Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead. - -Rarely you may also need to override: - -* `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default) -* `generate_roi_table` - for custom ROI optimization (if you need the ranges for the values in the ROI table that differ from default or the number of entries (steps) in the ROI table which differs from the default 4 steps) -* `stoploss_space` - for custom stoploss optimization (if you need the range for the stoploss parameter in the optimization hyperspace that differs from default) -* `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stop parameters in the optimization hyperspace that differ from default) - -#### Defining a buy signal optimization - -Let's say you are curious: should you use MACD crossings or lower Bollinger -Bands to trigger your buys. And you also wonder should you use RSI or ADX to -help with those buy decisions. If you decide to use RSI or ADX, which values -should I use for them? So let's use hyperparameter optimization to solve this -mystery. - -We will start by defining a search space: - -```python - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching strategy parameters - """ - return [ - Integer(20, 40, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal'], name='trigger') - ] -``` - -Above definition says: I have five parameters I want you to randomly combine -to find the best combination. Two of them are integer values (`adx-value` and `rsi-value`) and I want you test in the range of values 20 to 40. -Then we have three category variables. First two are either `True` or `False`. -We use these to either enable or disable the ADX and RSI guards. -The last one we call `trigger` and use it to decide which buy trigger we want to use. - -So let's write the buy strategy generator using these values: - -```python - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - conditions = [] - # GUARDS AND TRENDS - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend -``` - -Hyperopt will now call `populate_buy_trend()` many times (`epochs`) with different value combinations. -It will use the given historical data and make buys based on the buy signals generated with the above function. -Based on the results, hyperopt will tell you which parameter combination produced the best results (based on the configured [loss function](#loss-functions)). - -!!! Note - The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. - When you want to test an indicator that isn't used by the bot currently, remember to - add it to the `populate_indicators()` method in your strategy or hyperopt file. - -#### Sell optimization - -Similar to the buy-signal above, sell-signals can also be optimized. -Place the corresponding settings into the following methods - -* Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing. -* Within `sell_strategy_generator()` - populate the nested method `populate_sell_trend()` to apply the parameters. - -The configuration and rules are the same than for buy signals. -To avoid naming collisions in the search-space, please prefix all sell-spaces with `sell-`. - -### Execute Hyperopt - -Once you have updated your hyperopt configuration you can run it. -Because hyperopt tries a lot of combinations to find the best parameters it will take time to get a good result. More time usually results in better results. - -We strongly recommend to use `screen` or `tmux` to prevent any connection loss. - -```bash -freqtrade hyperopt --config config.json --hyperopt --hyperopt-loss --strategy -e 500 --spaces all -``` - -Use `` as the name of the custom hyperopt used. - -The `-e` option will set how many evaluations hyperopt will do. Since hyperopt uses Bayesian search, running too many epochs at once may not produce greater results. Experience has shown that best results are usually not improving much after 500-1000 epochs. -Doing multiple runs (executions) with a few 1000 epochs and different random state will most likely produce different results. - -The `--spaces all` option determines that all possible parameters should be optimized. Possibilities are listed below. - -!!! Note - Hyperopt will store hyperopt results with the timestamp of the hyperopt start time. - Reading commands (`hyperopt-list`, `hyperopt-show`) can use `--hyperopt-filename ` to read and display older hyperopt results. - You can find a list of filenames with `ls -l user_data/hyperopt_results/`. - -#### Running Hyperopt using methods from a strategy - -Hyperopt can reuse `populate_indicators`, `populate_buy_trend`, `populate_sell_trend` from your strategy, assuming these methods are **not** in your custom hyperopt file, and a strategy is provided. - -```bash -freqtrade hyperopt --hyperopt AwesomeHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy AwesomeStrategy -``` - -### Understand the Hyperopt Result - -Once Hyperopt is completed you can use the result to create a new strategy. -Given the following result from hyperopt: - -``` -Best result: - - 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 - -Buy hyperspace params: -{ 'adx-value': 44, - 'rsi-value': 29, - 'adx-enabled': False, - 'rsi-enabled': True, - 'trigger': 'bb_lower'} -``` - -You should understand this result like: - -* The buy trigger that worked best was `bb_lower`. -* You should not use ADX because `adx-enabled: False`) -* You should **consider** using the RSI indicator (`rsi-enabled: True` and the best value is `29.0` (`rsi-value: 29.0`) - -You have to look inside your strategy file into `buy_strategy_generator()` -method, what those values match to. - -So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that translates to the following code block: - -```python -(dataframe['rsi'] < 29.0) -``` - -Translating your whole hyperopt result as the new buy-signal would then look like: - -```python -def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: - dataframe.loc[ - ( - (dataframe['rsi'] < 29.0) & # rsi-value - dataframe['close'] < dataframe['bb_lowerband'] # trigger - ), - 'buy'] = 1 - return dataframe -``` - -### Validate backtesting results - -Once the optimized parameters and conditions have been implemented into your strategy, you should backtest the strategy to make sure everything is working as expected. - -To achieve same results (number of trades, their durations, profit, etc.) than during Hyperopt, please use same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting. - -Should results not match, please double-check to make sure you transferred all conditions correctly. -Pay special care to the stoploss (and trailing stoploss) parameters, as these are often set in configuration files, which override changes to the strategy. -You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss` or `trailing_stop`). - -### Sharing methods with your strategy - -Hyperopt classes provide access to the Strategy via the `strategy` class attribute. -This can be a great way to reduce code duplication if used correctly, but will also complicate usage for inexperienced users. - -``` python -from pandas import DataFrame -from freqtrade.strategy.interface import IStrategy -import freqtrade.vendor.qtpylib.indicators as qtpylib - -class MyAwesomeStrategy(IStrategy): - - buy_params = { - 'rsi-value': 30, - 'adx-value': 35, - } - - def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - return self.buy_strategy_generator(self.buy_params, dataframe, metadata) - - @staticmethod - def buy_strategy_generator(params, dataframe: DataFrame, metadata: dict) -> DataFrame: - dataframe.loc[ - ( - qtpylib.crossed_above(dataframe['rsi'], params['rsi-value']) & - dataframe['adx'] > params['adx-value']) & - dataframe['volume'] > 0 - ) - , 'buy'] = 1 - return dataframe - -class MyAwesomeHyperOpt(IHyperOpt): - ... - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - # Call strategy's buy strategy generator - return self.StrategyClass.buy_strategy_generator(params, dataframe, metadata) - - return populate_buy_trend -``` diff --git a/docs/bot-usage.md b/docs/bot-usage.md index b65220722..c6a7f6103 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -12,22 +12,22 @@ This page explains the different parameters of the bot and how to run it. ``` usage: freqtrade [-h] [-V] - {trade,create-userdir,new-config,new-hyperopt,new-strategy,download-data,convert-data,convert-trade-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,plot-dataframe,plot-profit} + {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} ... Free, open source crypto trading bot positional arguments: - {trade,create-userdir,new-config,new-hyperopt,new-strategy,download-data,convert-data,convert-trade-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,plot-dataframe,plot-profit} + {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} trade Trade module. create-userdir Create user-data directory. new-config Create new config - new-hyperopt Create new hyperopt new-strategy Create new strategy download-data Download backtesting data. convert-data Convert candle (OHLCV) data from one format to another. convert-trade-data Convert trade data from one format to another. + list-data List downloaded data. backtesting Backtesting module. edge Edge module. hyperopt Hyperopt module. @@ -41,8 +41,10 @@ positional arguments: list-timeframes Print available timeframes for the exchange. show-trades Show trades. test-pairlist Test your pairlist configuration. + install-ui Install FreqUI plot-dataframe Plot candles with indicators. plot-profit Generate plot showing profits. + webserver Webserver module. optional arguments: -h, --help show this help message and exit diff --git a/docs/deprecated.md b/docs/deprecated.md index b7ad847e6..d86a7ac7a 100644 --- a/docs/deprecated.md +++ b/docs/deprecated.md @@ -38,3 +38,8 @@ Since only quoteVolume can be compared between assets, the other options (bidVol Using `order_book_min` and `order_book_max` used to allow stepping the orderbook and trying to find the next ROI slot - trying to place sell-orders early. As this does however increase risk and provides no benefit, it's been removed for maintainability purposes in 2021.7. + +### Legacy Hyperopt mode + +Using separate hyperopt files was deprecated in 2021.4 and was removed in 2021.9. +Please switch to the new [Parametrized Strategies](hyperopt.md) to benefit from the new hyperopt interface. diff --git a/docs/faq.md b/docs/faq.md index b8a3a44d8..285625491 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -167,7 +167,7 @@ Since hyperopt uses Bayesian search, running for too many epochs may not produce It's therefore recommended to run between 500-1000 epochs over and over until you hit at least 10.000 epochs in total (or are satisfied with the result). You can best judge by looking at the results - if the bot keeps discovering better strategies, it's best to keep on going. ```bash -freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy SampleStrategy -e 1000 +freqtrade hyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy SampleStrategy -e 1000 ``` ### Why does it take a long time to run hyperopt? diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 1eb90f1bc..e69b761c4 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -44,9 +44,8 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--data-format-ohlcv {json,jsongz,hdf5}] [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] - [-p PAIRS [PAIRS ...]] [--hyperopt NAME] - [--hyperopt-path PATH] [--eps] [--dmmp] - [--enable-protections] + [-p PAIRS [PAIRS ...]] [--hyperopt-path PATH] + [--eps] [--dmmp] [--enable-protections] [--dry-run-wallet DRY_RUN_WALLET] [-e INT] [--spaces {all,buy,sell,roi,stoploss,trailing,protection,default} [{all,buy,sell,roi,stoploss,trailing,protection,default} ...]] [--print-all] [--no-color] [--print-json] [-j JOBS] @@ -73,10 +72,8 @@ optional arguments: -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] Limit command to these pairs. Pairs are space- separated. - --hyperopt NAME Specify hyperopt class name which will be used by the - bot. - --hyperopt-path PATH Specify additional lookup path for Hyperopt and - Hyperopt Loss functions. + --hyperopt-path PATH Specify additional lookup path for Hyperopt Loss + functions. --eps, --enable-position-stacking Allow buying the same pair multiple times (position stacking). @@ -558,7 +555,7 @@ For example, to use one month of data, pass `--timerange 20210101-20210201` (fro Full command: ```bash -freqtrade hyperopt --hyperopt --strategy --timerange 20210101-20210201 +freqtrade hyperopt --strategy --timerange 20210101-20210201 ``` ### Running Hyperopt with Smaller Search Space @@ -684,7 +681,7 @@ If you have the `generate_roi_table()` and `roi_space()` methods in your custom Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps). -A sample for these methods can be found in [sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). +A sample for these methods can be found in the [overriding pre-defined spaces section](advanced-hyperopt.md#overriding-pre-defined-spaces). !!! Note "Reduced search space" To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs. @@ -726,7 +723,7 @@ If you are optimizing stoploss values, Freqtrade creates the 'stoploss' optimiza If you have the `stoploss_space()` method in your custom hyperopt file, remove it in order to utilize Stoploss hyperoptimization space generated by Freqtrade by default. -Override the `stoploss_space()` method and define the desired range in it if you need stoploss values to vary in other range during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). +Override the `stoploss_space()` method and define the desired range in it if you need stoploss values to vary in other range during hyperoptimization. A sample for this method can be found in the [overriding pre-defined spaces section](advanced-hyperopt.md#overriding-pre-defined-spaces). !!! Note "Reduced search space" To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs. @@ -764,10 +761,10 @@ As stated in the comment, you can also use it as the values of the corresponding If you are optimizing trailing stop values, Freqtrade creates the 'trailing' optimization hyperspace for you. By default, the `trailing_stop` parameter is always set to True in that hyperspace, the value of the `trailing_only_offset_is_reached` vary between True and False, the values of the `trailing_stop_positive` and `trailing_stop_positive_offset` parameters vary in the ranges 0.02...0.35 and 0.01...0.1 correspondingly, which is sufficient in most cases. -Override the `trailing_space()` method and define the desired range in it if you need values of the trailing stop parameters to vary in other ranges during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). +Override the `trailing_space()` method and define the desired range in it if you need values of the trailing stop parameters to vary in other ranges during hyperoptimization. A sample for this method can be found in the [overriding pre-defined spaces section](advanced-hyperopt.md#overriding-pre-defined-spaces). !!! Note "Reduced search space" - To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs. + To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#overriding-pre-defined-spaces) to change this to your needs. ### Reproducible results diff --git a/docs/utils.md b/docs/utils.md index a1e219d6a..ade2f4047 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -26,9 +26,7 @@ optional arguments: ├── data ├── hyperopt_results ├── hyperopts -│   ├── sample_hyperopt_advanced.py │   ├── sample_hyperopt_loss.py -│   └── sample_hyperopt.py ├── notebooks │   └── strategy_analysis_example.ipynb ├── plot diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 9c6ce543f..3574dc485 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -90,7 +90,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-data", - "list-hyperopts", "hyperopt-list", "hyperopt-show", + "hyperopt-list", "hyperopt-show", "plot-dataframe", "plot-profit", "show-trades"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index a1790cb9a..e3c7fe464 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -1,7 +1,7 @@ """ Definition of cli arguments used in arguments.py """ -from argparse import ArgumentTypeError +from argparse import SUPPRESS, ArgumentTypeError from freqtrade import __version__, constants from freqtrade.constants import HYPEROPT_LOSS_BUILTIN @@ -203,7 +203,7 @@ AVAILABLE_CLI_OPTIONS = { # Hyperopt "hyperopt": Arg( '--hyperopt', - help='Specify hyperopt class name which will be used by the bot.', + help=SUPPRESS, metavar='NAME', required=False, ), diff --git a/freqtrade/constants.py b/freqtrade/constants.py index efcd1aaca..9ca43d459 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -69,9 +69,7 @@ DUST_PER_COIN = { # Source files with destination directories within user-directory USER_DATA_FILES = { 'sample_strategy.py': USERPATH_STRATEGIES, - 'sample_hyperopt_advanced.py': USERPATH_HYPEROPTS, 'sample_hyperopt_loss.py': USERPATH_HYPEROPTS, - 'sample_hyperopt.py': USERPATH_HYPEROPTS, 'strategy_analysis_example.ipynb': USERPATH_NOTEBOOKS, } diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 90e19782e..42a10e8d7 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -185,7 +185,7 @@ def test_exchange_resolver(default_conf, mocker, caplog): def test_validate_order_time_in_force(default_conf, mocker, caplog): caplog.set_level(logging.INFO) - # explicitly test bittrex, exchanges implementing other policies need seperate tests + # explicitly test bittrex, exchanges implementing other policies need separate tests ex = get_patched_exchange(mocker, default_conf, id="bittrex") tif = { "buy": "gtc", @@ -2464,7 +2464,7 @@ def test_fetch_order(default_conf, mocker, exchange_name, caplog): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_fetch_stoploss_order(default_conf, mocker, exchange_name): - # Don't test FTX here - that needs a seperate test + # Don't test FTX here - that needs a separate test if exchange_name == 'ftx': return default_conf['dry_run'] = True diff --git a/tests/optimize/hyperopts/hyperopt_test_sep_file.py b/tests/optimize/hyperopts/hyperopt_test_sep_file.py deleted file mode 100644 index 0fa1e1959..000000000 --- a/tests/optimize/hyperopts/hyperopt_test_sep_file.py +++ /dev/null @@ -1,202 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement - -from functools import reduce -from typing import Any, Callable, Dict, List - -import talib.abstract as ta -from pandas import DataFrame -from skopt.space import Categorical, Dimension, Integer - -import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.optimize.hyperopt_interface import IHyperOpt - - -class HyperoptTestSepFile(IHyperOpt): - """ - Default hyperopt provided by the Freqtrade bot. - You can override it with your own Hyperopt - """ - @staticmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Add several indicators needed for buy and sell strategies defined below. - """ - # ADX - dataframe['adx'] = ta.ADX(dataframe) - # MACD - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - # MFI - dataframe['mfi'] = ta.MFI(dataframe) - # RSI - dataframe['rsi'] = ta.RSI(dataframe) - # Stochastic Fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - # Minus-DI - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_upperband'] = bollinger['upper'] - # SAR - dataframe['sar'] = ta.SAR(dataframe) - - return dataframe - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - if 'mfi-enabled' in params and params['mfi-enabled']: - conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd-enabled' in params and params['fastd-enabled']: - conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching buy strategy parameters. - """ - return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') - ] - - @staticmethod - def sell_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the sell strategy parameters to be used by Hyperopt. - """ - def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Sell strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: - conditions.append(dataframe['mfi'] > params['sell-mfi-value']) - if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: - conditions.append(dataframe['fastd'] > params['sell-fastd-value']) - if 'sell-adx-enabled' in params and params['sell-adx-enabled']: - conditions.append(dataframe['adx'] < params['sell-adx-value']) - if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) - - # TRIGGERS - if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_upper': - conditions.append(dataframe['close'] > dataframe['bb_upperband']) - if params['sell-trigger'] == 'sell-macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) - if params['sell-trigger'] == 'sell-sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['sar'], dataframe['close'] - )) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 - - return dataframe - - return populate_sell_trend - - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters. - """ - return [ - Integer(75, 100, name='sell-mfi-value'), - Integer(50, 100, name='sell-fastd-value'), - Integer(50, 100, name='sell-adx-value'), - Integer(60, 100, name='sell-rsi-value'), - Categorical([True, False], name='sell-mfi-enabled'), - Categorical([True, False], name='sell-fastd-enabled'), - Categorical([True, False], name='sell-adx-enabled'), - Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') - ] - - def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Based on TA indicators. Should be a copy of same method from strategy. - Must align to populate_indicators in this file. - Only used when --spaces does not include buy space. - """ - dataframe.loc[ - ( - (dataframe['close'] < dataframe['bb_lowerband']) & - (dataframe['mfi'] < 16) & - (dataframe['adx'] > 25) & - (dataframe['rsi'] < 21) - ), - 'buy'] = 1 - - return dataframe - - def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Based on TA indicators. Should be a copy of same method from strategy. - Must align to populate_indicators in this file. - Only used when --spaces does not include sell space. - """ - dataframe.loc[ - ( - (qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) & - (dataframe['fastd'] > 54) - ), - 'sell'] = 1 - - return dataframe diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 498b6e588..b34c3a916 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -21,8 +21,6 @@ from freqtrade.strategy.hyper import IntParameter from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) -from .hyperopts.hyperopt_test_sep_file import HyperoptTestSepFile - def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) @@ -30,7 +28,7 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', ] config = setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) @@ -62,7 +60,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', '--datadir', '/foo/bar', '--timeframe', '1m', '--timerange', ':100', @@ -114,7 +112,7 @@ def test_setup_hyperopt_configuration_stake_amount(mocker, default_conf) -> None args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', '--stake-amount', '1', '--starting-balance', '2' ] @@ -142,9 +140,7 @@ def test_start_not_installed(mocker, default_conf, import_fails) -> None: args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', - '--hyperopt-path', - str(Path(__file__).parent / "hyperopts"), + '--strategy', 'HyperoptableStrategy', '--epochs', '5', '--hyperopt-loss', 'SharpeHyperOptLossDaily', ] @@ -203,7 +199,7 @@ def test_start_filelock(mocker, hyperopt_conf, caplog) -> None: args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', '--hyperopt-loss', 'SharpeHyperOptLossDaily', '--epochs', '5' ] diff --git a/tests/plugins/test_pairlocks.py b/tests/plugins/test_pairlocks.py index fce3a8cd1..c694fd7c1 100644 --- a/tests/plugins/test_pairlocks.py +++ b/tests/plugins/test_pairlocks.py @@ -68,7 +68,7 @@ def test_PairLocks(use_db): # Global lock PairLocks.lock_pair('*', lock_time) assert PairLocks.is_global_lock(lock_time + timedelta(minutes=-50)) - # Global lock also locks every pair seperately + # Global lock also locks every pair separately assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-50)) assert PairLocks.is_pair_locked('XRP/USDT', lock_time + timedelta(minutes=-50)) diff --git a/tests/test_directory_operations.py b/tests/test_directory_operations.py index a11200526..905b078f9 100644 --- a/tests/test_directory_operations.py +++ b/tests/test_directory_operations.py @@ -74,16 +74,12 @@ def test_copy_sample_files(mocker, default_conf, caplog) -> None: copymock = mocker.patch('shutil.copy', MagicMock()) copy_sample_files(Path('/tmp/bar')) - assert copymock.call_count == 5 + assert copymock.call_count == 3 assert copymock.call_args_list[0][0][1] == str( Path('/tmp/bar') / 'strategies/sample_strategy.py') assert copymock.call_args_list[1][0][1] == str( - Path('/tmp/bar') / 'hyperopts/sample_hyperopt_advanced.py') - assert copymock.call_args_list[2][0][1] == str( Path('/tmp/bar') / 'hyperopts/sample_hyperopt_loss.py') - assert copymock.call_args_list[3][0][1] == str( - Path('/tmp/bar') / 'hyperopts/sample_hyperopt.py') - assert copymock.call_args_list[4][0][1] == str( + assert copymock.call_args_list[2][0][1] == str( Path('/tmp/bar') / 'notebooks/strategy_analysis_example.ipynb') From 0017b3438efcf10fad1ac0224db6c7d4c6d25a0a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Sep 2021 17:56:41 +0200 Subject: [PATCH 065/239] Remove list-hyperopts --- docs/utils.md | 46 ++++------------------------- freqtrade/commands/__init__.py | 6 ++-- freqtrade/commands/arguments.py | 19 ++++-------- freqtrade/commands/list_commands.py | 21 +------------ 4 files changed, 14 insertions(+), 78 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index ade2f4047..d8fbcacb7 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -109,11 +109,11 @@ Using the advanced template (populates all optional functions and methods) freqtrade new-strategy --strategy AwesomeStrategy --template advanced ``` -## List Strategies and List Hyperopts +## List Strategies -Use the `list-strategies` subcommand to see all strategies in one particular directory and the `list-hyperopts` subcommand to list custom Hyperopts. +Use the `list-strategies` subcommand to see all strategies in one particular directory. -These subcommands are useful for finding problems in your environment with loading strategies or hyperopt classes: modules with strategies or hyperopt classes that contain errors and failed to load are printed in red (LOAD FAILED), while strategies or hyperopt classes with duplicate names are printed in yellow (DUPLICATE NAME). +This subcommand is useful for finding problems in your environment with loading strategies: modules with strategies that contain errors and failed to load are printed in red (LOAD FAILED), while strategies with duplicate names are printed in yellow (DUPLICATE NAME). ``` usage: freqtrade list-strategies [-h] [-v] [--logfile FILE] [-V] [-c PATH] @@ -127,34 +127,6 @@ optional arguments: --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. -Common arguments: - -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. Special values are: - 'syslog', 'journald'. See the documentation for more - details. - -V, --version show program's version number and exit - -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. - -d PATH, --datadir PATH - Path to directory with historical backtesting data. - --userdir PATH, --user-data-dir PATH - Path to userdata directory. -``` -``` -usage: freqtrade list-hyperopts [-h] [-v] [--logfile FILE] [-V] [-c PATH] - [-d PATH] [--userdir PATH] - [--hyperopt-path PATH] [-1] [--no-color] - -optional arguments: - -h, --help show this help message and exit - --hyperopt-path PATH Specify additional lookup path for Hyperopt and - Hyperopt Loss functions. - -1, --one-column Print output in one column. - --no-color Disable colorization of hyperopt results. May be - useful if you are redirecting output to a file. - Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). --logfile FILE Log to the file specified. Special values are: @@ -174,18 +146,16 @@ Common arguments: !!! Warning Using these commands will try to load all python files from a directory. This can be a security risk if untrusted files reside in this directory, since all module-level code is executed. -Example: Search default strategies and hyperopts directories (within the default userdir). +Example: Search default strategies directories (within the default userdir). ``` bash freqtrade list-strategies -freqtrade list-hyperopts ``` -Example: Search strategies and hyperopts directory within the userdir. +Example: Search strategies directory within the userdir. ``` bash freqtrade list-strategies --userdir ~/.freqtrade/ -freqtrade list-hyperopts --userdir ~/.freqtrade/ ``` Example: Search dedicated strategy path. @@ -194,12 +164,6 @@ Example: Search dedicated strategy path. freqtrade list-strategies --strategy-path ~/.freqtrade/strategies/ ``` -Example: Search dedicated hyperopt path. - -``` bash -freqtrade list-hyperopt --hyperopt-path ~/.freqtrade/hyperopts/ -``` - ## List Exchanges Use the `list-exchanges` subcommand to see the exchanges available for the bot. diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 49b184a80..a6f14cff7 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -13,9 +13,9 @@ from freqtrade.commands.data_commands import (start_convert_data, start_download from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui, start_new_strategy) from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show -from freqtrade.commands.list_commands import (start_list_exchanges, start_list_hyperopts, - start_list_markets, start_list_strategies, - start_list_timeframes, start_show_trades) +from freqtrade.commands.list_commands import (start_list_exchanges, start_list_markets, + start_list_strategies, start_list_timeframes, + start_show_trades) from freqtrade.commands.optimize_commands import start_backtesting, start_edge, start_hyperopt from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 3574dc485..d424f3ce7 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -172,11 +172,11 @@ class Arguments: from freqtrade.commands import (start_backtesting, start_convert_data, start_create_userdir, start_download_data, start_edge, start_hyperopt, start_hyperopt_list, start_hyperopt_show, start_install_ui, - start_list_data, start_list_exchanges, start_list_hyperopts, - start_list_markets, start_list_strategies, - start_list_timeframes, start_new_config, start_new_strategy, - start_plot_dataframe, start_plot_profit, start_show_trades, - start_test_pairlist, start_trading, start_webserver) + start_list_data, start_list_exchanges, start_list_markets, + start_list_strategies, start_list_timeframes, + start_new_config, start_new_strategy, start_plot_dataframe, + start_plot_profit, start_show_trades, start_test_pairlist, + start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -291,15 +291,6 @@ class Arguments: list_exchanges_cmd.set_defaults(func=start_list_exchanges) self._build_args(optionlist=ARGS_LIST_EXCHANGES, parser=list_exchanges_cmd) - # Add list-hyperopts subcommand - list_hyperopts_cmd = subparsers.add_parser( - 'list-hyperopts', - help='Print available hyperopt classes.', - parents=[_common_parser], - ) - list_hyperopts_cmd.set_defaults(func=start_list_hyperopts) - self._build_args(optionlist=ARGS_LIST_HYPEROPTS, parser=list_hyperopts_cmd) - # Add list-markets subcommand list_markets_cmd = subparsers.add_parser( 'list-markets', diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 410b9b72b..38fb098a0 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -10,7 +10,7 @@ from colorama import init as colorama_init from tabulate import tabulate from freqtrade.configuration import setup_utils_configuration -from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES +from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import market_is_active, validate_exchanges @@ -92,25 +92,6 @@ def start_list_strategies(args: Dict[str, Any]) -> None: _print_objs_tabular(strategy_objs, config.get('print_colorized', False)) -def start_list_hyperopts(args: Dict[str, Any]) -> None: - """ - Print files with HyperOpt custom classes available in the directory - """ - from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - directory = Path(config.get('hyperopt_path', config['user_data_dir'] / USERPATH_HYPEROPTS)) - hyperopt_objs = HyperOptResolver.search_all_objects(directory, not args['print_one_column']) - # Sort alphabetically - hyperopt_objs = sorted(hyperopt_objs, key=lambda x: x['name']) - - if args['print_one_column']: - print('\n'.join([s['name'] for s in hyperopt_objs])) - else: - _print_objs_tabular(hyperopt_objs, config.get('print_colorized', False)) - - def start_list_timeframes(args: Dict[str, Any]) -> None: """ Print timeframes available on Exchange From 236dc4800075a30a4c7ed1e52ef45be0d785ef76 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Sep 2021 07:46:17 +0200 Subject: [PATCH 066/239] Update CI to use new hyperopt interface --- .github/workflows/ci.yml | 6 +++--- .travis.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb767efb1..228a60389 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,7 +87,7 @@ jobs: run: | cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - name: Flake8 run: | @@ -180,7 +180,7 @@ jobs: run: | cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - name: Flake8 run: | @@ -247,7 +247,7 @@ jobs: run: | cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - name: Flake8 run: | diff --git a/.travis.yml b/.travis.yml index f2a6d508d..15c174bfe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ jobs: - script: - cp config_examples/config_bittrex.example.json config.json - freqtrade create-userdir --userdir user_data - - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily + - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily name: hyperopt - script: flake8 name: flake8 From d8f48cf0e3b7376ec8047a0a22e525175cb5fc98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 03:01:14 +0000 Subject: [PATCH 067/239] Bump pandas from 1.3.2 to 1.3.3 Bumps [pandas](https://github.com/pandas-dev/pandas) from 1.3.2 to 1.3.3. - [Release notes](https://github.com/pandas-dev/pandas/releases) - [Changelog](https://github.com/pandas-dev/pandas/blob/master/RELEASE.md) - [Commits](https://github.com/pandas-dev/pandas/compare/v1.3.2...v1.3.3) --- updated-dependencies: - dependency-name: pandas dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e2bed0f9e..9e4ec97d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ numpy==1.21.2 -pandas==1.3.2 +pandas==1.3.3 ccxt==1.55.83 # Pin cryptography for now due to rust build errors with piwheels From 81039fce28dbae0fcea77a4ab952fb91a847d5aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 03:01:24 +0000 Subject: [PATCH 068/239] Bump progressbar2 from 3.53.1 to 3.53.2 Bumps [progressbar2](https://github.com/WoLpH/python-progressbar) from 3.53.1 to 3.53.2. - [Release notes](https://github.com/WoLpH/python-progressbar/releases) - [Changelog](https://github.com/WoLpH/python-progressbar/blob/develop/CHANGES.rst) - [Commits](https://github.com/WoLpH/python-progressbar/compare/v3.53.1...v3.53.2) --- updated-dependencies: - dependency-name: progressbar2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index d7f22634b..7dc55a9fc 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -8,4 +8,4 @@ scikit-optimize==0.8.1 filelock==3.0.12 joblib==1.0.1 psutil==5.8.0 -progressbar2==3.53.1 +progressbar2==3.53.2 From b13bd87625d122278644f8210d07b8cad13cd352 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 04:31:52 +0000 Subject: [PATCH 069/239] Bump ccxt from 1.55.83 to 1.56.30 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.55.83 to 1.56.30. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.55.83...1.56.30) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9e4ec97d6..aa729dd9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.21.2 pandas==1.3.3 -ccxt==1.55.83 +ccxt==1.56.30 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.8 aiohttp==3.7.4.post0 From 3cdd06f5627b7baee7b5ec3ead09e036080e9b66 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Sep 2021 19:32:51 +0200 Subject: [PATCH 070/239] Add PeriodicCache --- freqtrade/configuration/PeriodicCache.py | 19 +++++++++++++++ freqtrade/configuration/__init__.py | 1 + requirements-dev.txt | 2 ++ tests/test_periodiccache.py | 31 ++++++++++++++++++++++++ 4 files changed, 53 insertions(+) create mode 100644 freqtrade/configuration/PeriodicCache.py create mode 100644 tests/test_periodiccache.py diff --git a/freqtrade/configuration/PeriodicCache.py b/freqtrade/configuration/PeriodicCache.py new file mode 100644 index 000000000..25c0c47f3 --- /dev/null +++ b/freqtrade/configuration/PeriodicCache.py @@ -0,0 +1,19 @@ +from datetime import datetime, timezone + +from cachetools.ttl import TTLCache + + +class PeriodicCache(TTLCache): + """ + Special cache that expires at "straight" times + A timer with ttl of 3600 (1h) will expire at every full hour (:00). + """ + + def __init__(self, maxsize, ttl, getsizeof=None): + def local_timer(): + ts = datetime.now(timezone.utc).timestamp() + offset = (ts % ttl) + return ts - offset + + # Init with smlight offset + super().__init__(maxsize=maxsize, ttl=ttl-1e-5, timer=local_timer, getsizeof=getsizeof) diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 607f9cdef..b1b268092 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa: F401 +from freqtrade.configuration.PeriodicCache import PeriodicCache from freqtrade.configuration.check_exchange import check_exchange, remove_credentials from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.config_validation import validate_config_consistency diff --git a/requirements-dev.txt b/requirements-dev.txt index 34d5607f3..4859e1cc6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,6 +14,8 @@ pytest-cov==2.12.1 pytest-mock==3.6.1 pytest-random-order==1.0.4 isort==5.9.3 +# For datetime mocking +time-machine==2.4.0 # Convert jupyter notebooks to markdown documents nbconvert==6.1.0 diff --git a/tests/test_periodiccache.py b/tests/test_periodiccache.py new file mode 100644 index 000000000..ff9b53684 --- /dev/null +++ b/tests/test_periodiccache.py @@ -0,0 +1,31 @@ +from freqtrade.configuration import PeriodicCache +import time_machine + + +def test_ttl_cache(): + + with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: + + cache = PeriodicCache(5, ttl=60) + cache1h = PeriodicCache(5, ttl=3600) + + assert cache.timer() == 1630472400.0 + cache['a'] = 1235 + cache1h['a'] = 555123 + assert 'a' in cache + assert 'a' in cache1h + + t.move_to("2021-09-01 05:00:59 +00:00") + assert 'a' in cache + assert 'a' in cache1h + + # Cache expired + t.move_to("2021-09-01 05:01:00 +00:00") + assert 'a' not in cache + assert 'a' in cache1h + + t.move_to("2021-09-01 05:59:59 +00:00") + assert 'a' in cache1h + + t.move_to("2021-09-01 06:00:00 +00:00") + assert 'a' not in cache1h From 8afb3c4b70c3458772953c8be23976bd90a264ba Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Sep 2021 19:33:28 +0200 Subject: [PATCH 071/239] Move AgeFilter cache to instance level --- freqtrade/plugins/pairlist/AgeFilter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index dc5cab31e..32fef5fb0 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -18,14 +18,14 @@ logger = logging.getLogger(__name__) class AgeFilter(IPairList): - # Checked symbols cache (dictionary of ticker symbol => timestamp) - _symbolsChecked: Dict[str, int] = {} - def __init__(self, exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + # Checked symbols cache (dictionary of ticker symbol => timestamp) + self._symbolsChecked: Dict[str, int] = {} + self._min_days_listed = pairlistconfig.get('min_days_listed', 10) self._max_days_listed = pairlistconfig.get('max_days_listed', None) From a12c3ecc9b2c7fa005f4104f22cde5cba23829b6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Sep 2021 20:00:22 +0200 Subject: [PATCH 072/239] Remove credentials whenever dry-run is set from within the exchange --- freqtrade/configuration/__init__.py | 2 +- freqtrade/configuration/check_exchange.py | 13 ------------- freqtrade/configuration/config_setup.py | 5 ++--- freqtrade/exchange/__init__.py | 2 +- freqtrade/exchange/common.py | 13 +++++++++++++ freqtrade/exchange/exchange.py | 5 +++-- freqtrade/optimize/backtesting.py | 5 ++--- freqtrade/optimize/edge_cli.py | 6 +++--- tests/commands/test_commands.py | 2 -- tests/exchange/test_ccxt_compat.py | 2 +- tests/exchange/test_exchange.py | 19 ++++++++++++++++++- tests/test_configuration.py | 15 +-------------- 12 files changed, 45 insertions(+), 44 deletions(-) diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 607f9cdef..730a4e47f 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,6 +1,6 @@ # flake8: noqa: F401 -from freqtrade.configuration.check_exchange import check_exchange, remove_credentials +from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.config_validation import validate_config_consistency from freqtrade.configuration.configuration import Configuration diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index c4f038103..fa1f47f9b 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -10,19 +10,6 @@ from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt, logger = logging.getLogger(__name__) -def remove_credentials(config: Dict[str, Any]) -> None: - """ - Removes exchange keys from the configuration and specifies dry-run - Used for backtesting / hyperopt / edge and utils. - Modifies the input dict! - """ - config['exchange']['key'] = '' - config['exchange']['secret'] = '' - config['exchange']['password'] = '' - config['exchange']['uid'] = '' - config['dry_run'] = True - - def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool: """ Check if the exchange name in the config file is supported by Freqtrade diff --git a/freqtrade/configuration/config_setup.py b/freqtrade/configuration/config_setup.py index 22836ab19..02f2d4089 100644 --- a/freqtrade/configuration/config_setup.py +++ b/freqtrade/configuration/config_setup.py @@ -3,7 +3,6 @@ from typing import Any, Dict from freqtrade.enums import RunMode -from .check_exchange import remove_credentials from .config_validation import validate_config_consistency from .configuration import Configuration @@ -21,8 +20,8 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str configuration = Configuration(args, method) config = configuration.get_config() - # Ensure we do not use Exchange credentials - remove_credentials(config) + # Ensure these modes are using Dry-run + config['dry_run'] = True validate_config_consistency(config) return config diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index b0c88a51a..b08213d28 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -1,6 +1,6 @@ # flake8: noqa: F401 # isort: off -from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS +from freqtrade.exchange.common import remove_credentials, MAP_EXCHANGE_CHILDCLASS from freqtrade.exchange.exchange import Exchange # isort: on from freqtrade.exchange.bibox import Bibox diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index 694aa3aa2..7b89adf06 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -51,6 +51,19 @@ EXCHANGE_HAS_OPTIONAL = [ ] +def remove_credentials(config) -> None: + """ + Removes exchange keys from the configuration and specifies dry-run + Used for backtesting / hyperopt / edge and utils. + Modifies the input dict! + """ + if config.get('dry_run', False): + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + config['exchange']['password'] = '' + config['exchange']['uid'] = '' + + def calculate_backoff(retrycount, max_retries): """ Calculate backoff diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 11de5411f..2b9b08d70 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -26,8 +26,8 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun InvalidOrderException, OperationalException, PricingError, RetryableOrderError, TemporaryError) from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, - EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier, - retrier_async) + EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, + remove_credentials, retrier, retrier_async) from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -104,6 +104,7 @@ class Exchange: # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} + remove_credentials(config) if config['dry_run']: logger.info('Instance is running with dry_run enabled') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4b52e104b..3e06bfa1b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame -from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency +from freqtrade.configuration import TimeRange, validate_config_consistency from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.data import history from freqtrade.data.btanalysis import trade_list_to_dataframe @@ -61,8 +61,7 @@ class Backtesting: self.config = config self.results: Optional[Dict[str, Any]] = None - # Reset keys for backtesting - remove_credentials(self.config) + config['dry_run'] = True self.strategylist: List[IStrategy] = [] self.all_results: Dict[str, Dict] = {} diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index aab7def05..417faa685 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -7,7 +7,7 @@ import logging from typing import Any, Dict from freqtrade import constants -from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency +from freqtrade.configuration import TimeRange, validate_config_consistency from freqtrade.edge import Edge from freqtrade.optimize.optimize_reports import generate_edge_table from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -28,8 +28,8 @@ class EdgeCli: def __init__(self, config: Dict[str, Any]) -> None: self.config = config - # Reset keys for edge - remove_credentials(self.config) + # Ensure using dry-run + self.config['dry_run'] = True self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) self.strategy = StrategyResolver.load_strategy(self.config) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 1da9e5100..87040bfd5 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -32,8 +32,6 @@ def test_setup_utils_configuration(): config = setup_utils_configuration(get_args(args), RunMode.OTHER) assert "exchange" in config assert config['dry_run'] is True - assert config['exchange']['key'] == '' - assert config['exchange']['secret'] == '' def test_start_trading_fail(mocker, caplog): diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 3a32d108b..0f7963e36 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -54,7 +54,7 @@ EXCHANGES = { def exchange_conf(): config = get_default_conf((Path(__file__).parent / "testdata").resolve()) config['exchange']['pair_whitelist'] = [] - config['dry_run'] = False + # config['dry_run'] = False return config diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 90e19782e..baf1f081a 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1,5 +1,6 @@ import copy import logging +from copy import deepcopy from datetime import datetime, timedelta, timezone from math import isclose from random import randint @@ -14,7 +15,7 @@ from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOr OperationalException, PricingError, TemporaryError) from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT, - calculate_backoff) + calculate_backoff, remove_credentials) from freqtrade.exchange.exchange import (market_is_active, timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date, timeframe_to_seconds) @@ -78,6 +79,22 @@ def test_init(default_conf, mocker, caplog): assert log_has('Instance is running with dry_run enabled', caplog) +def test_remove_credentials(default_conf, caplog) -> None: + conf = deepcopy(default_conf) + conf['dry_run'] = False + remove_credentials(conf) + + assert conf['exchange']['key'] != '' + assert conf['exchange']['secret'] != '' + + conf['dry_run'] = True + remove_credentials(conf) + assert conf['exchange']['key'] == '' + assert conf['exchange']['secret'] == '' + assert conf['exchange']['password'] == '' + assert conf['exchange']['uid'] == '' + + def test_init_ccxt_kwargs(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 9aea4fa11..1ce45e4d5 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -11,8 +11,7 @@ import pytest from jsonschema import ValidationError from freqtrade.commands import Arguments -from freqtrade.configuration import (Configuration, check_exchange, remove_credentials, - validate_config_consistency) +from freqtrade.configuration import Configuration, check_exchange, validate_config_consistency from freqtrade.configuration.config_validation import validate_config_schema from freqtrade.configuration.deprecated_settings import (check_conflicting_settings, process_deprecated_setting, @@ -617,18 +616,6 @@ def test_check_exchange(default_conf, caplog) -> None: check_exchange(default_conf) -def test_remove_credentials(default_conf, caplog) -> None: - conf = deepcopy(default_conf) - conf['dry_run'] = False - remove_credentials(conf) - - assert conf['dry_run'] is True - assert conf['exchange']['key'] == '' - assert conf['exchange']['secret'] == '' - assert conf['exchange']['password'] == '' - assert conf['exchange']['uid'] == '' - - def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) From c9ba52d7321b2cae17776c66723c837a75c3bfdf Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Sep 2021 06:30:18 +0200 Subject: [PATCH 073/239] Expire cached pairs in age-filter once per day --- freqtrade/plugins/pairlist/AgeFilter.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 32fef5fb0..1fba00123 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -11,6 +11,7 @@ from pandas import DataFrame from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.configuration import PeriodicCache logger = logging.getLogger(__name__) @@ -25,6 +26,7 @@ class AgeFilter(IPairList): # Checked symbols cache (dictionary of ticker symbol => timestamp) self._symbolsChecked: Dict[str, int] = {} + self._too_young_pairs = PeriodicCache(maxsize=1000, ttl=86_400) self._min_days_listed = pairlistconfig.get('min_days_listed', 10) self._max_days_listed = pairlistconfig.get('max_days_listed', None) @@ -69,10 +71,12 @@ class AgeFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new allowlist """ - needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked] + needed_pairs = [ + (p, '1d') for p in pairlist + if p not in self._symbolsChecked and p not in self._too_young_pairs] if not needed_pairs: return pairlist - + logger.info(f"needed pairs {needed_pairs}") since_days = -( self._max_days_listed if self._max_days_listed else self._min_days_listed ) - 1 @@ -118,5 +122,6 @@ class AgeFilter(IPairList): " or more than " f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" ) if self._max_days_listed else ''), logger.info) + self._too_young_pairs[pair] = arrow.utcnow().int_timestamp * 1000 return False return False From 3ce5197e8d07471e75be14df435c3d291b3b69c4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Sep 2021 06:45:26 +0200 Subject: [PATCH 074/239] Add Tests for AgeFilter caching closes #5552 --- freqtrade/configuration/__init__.py | 2 +- freqtrade/plugins/pairlist/AgeFilter.py | 11 ++-- tests/plugins/test_pairlist.py | 69 ++++++++++++++++--------- tests/test_periodiccache.py | 3 +- 4 files changed, 54 insertions(+), 31 deletions(-) diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index b1b268092..dccbb14b3 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,8 +1,8 @@ # flake8: noqa: F401 -from freqtrade.configuration.PeriodicCache import PeriodicCache from freqtrade.configuration.check_exchange import check_exchange, remove_credentials from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.config_validation import validate_config_consistency from freqtrade.configuration.configuration import Configuration +from freqtrade.configuration.PeriodicCache import PeriodicCache from freqtrade.configuration.timerange import TimeRange diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 1fba00123..c43bd0c4c 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -8,10 +8,10 @@ from typing import Any, Dict, List, Optional import arrow from pandas import DataFrame +from freqtrade.configuration import PeriodicCache from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList -from freqtrade.configuration import PeriodicCache logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ class AgeFilter(IPairList): # Checked symbols cache (dictionary of ticker symbol => timestamp) self._symbolsChecked: Dict[str, int] = {} - self._too_young_pairs = PeriodicCache(maxsize=1000, ttl=86_400) + self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400) self._min_days_listed = pairlistconfig.get('min_days_listed', 10) self._max_days_listed = pairlistconfig.get('max_days_listed', None) @@ -73,9 +73,10 @@ class AgeFilter(IPairList): """ needed_pairs = [ (p, '1d') for p in pairlist - if p not in self._symbolsChecked and p not in self._too_young_pairs] + if p not in self._symbolsChecked and p not in self._symbolsCheckFailed] if not needed_pairs: - return pairlist + # Remove pairs that have been removed before + return [p for p in pairlist if p not in self._symbolsCheckFailed] logger.info(f"needed pairs {needed_pairs}") since_days = -( self._max_days_listed if self._max_days_listed else self._min_days_listed @@ -122,6 +123,6 @@ class AgeFilter(IPairList): " or more than " f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" ) if self._max_days_listed else ''), logger.info) - self._too_young_pairs[pair] = arrow.utcnow().int_timestamp * 1000 + self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000 return False return False diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 5f0701a22..3cdf6d341 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -4,6 +4,7 @@ import time from unittest.mock import MagicMock, PropertyMock import pytest +import time_machine from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.exceptions import OperationalException @@ -815,32 +816,52 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history): - ohlcv_data = { - ('ETH/BTC', '1d'): ohlcv_history, - ('TKN/BTC', '1d'): ohlcv_history, - ('LTC/BTC', '1d'): ohlcv_history, - } - mocker.patch.multiple('freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=markets), - exchange_has=MagicMock(return_value=True), - get_tickers=tickers - ) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), - ) + with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + ('XRP/BTC', '1d'): ohlcv_history.iloc[[0]], + } + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers, + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), + ) - freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter) - assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0 - freqtrade.pairlists.refresh_pairlist() - assert len(freqtrade.pairlists.whitelist) == 3 - assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter) + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0 + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 - previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count - freqtrade.pairlists.refresh_pairlist() - assert len(freqtrade.pairlists.whitelist) == 3 - # Called once for XRP/BTC - assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1 + previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + # Call to XRP/BTC cached + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + # Move to next day + t.move_to("2021-09-02 01:00:00 +00:00") + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1 + + # Move another day with fresh mocks (now the pair is old enough) + t.move_to("2021-09-03 01:00:00 +00:00") + # Called once for XRP/BTC + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + ('XRP/BTC', '1d'): ohlcv_history, + } + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 4 + # Called once (only for XRP/BTC) + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1 def test_OffsetFilter_error(mocker, whitelist_conf) -> None: diff --git a/tests/test_periodiccache.py b/tests/test_periodiccache.py index ff9b53684..f874f9041 100644 --- a/tests/test_periodiccache.py +++ b/tests/test_periodiccache.py @@ -1,6 +1,7 @@ -from freqtrade.configuration import PeriodicCache import time_machine +from freqtrade.configuration import PeriodicCache + def test_ttl_cache(): From 35eda8c8c7c304051cae6b2a22453f56aa0ea4a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Sep 2021 07:07:20 +0200 Subject: [PATCH 075/239] Improve agefilter test --- tests/plugins/test_pairlist.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 3cdf6d341..34770c03d 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -821,7 +821,6 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o ('ETH/BTC', '1d'): ohlcv_history, ('TKN/BTC', '1d'): ohlcv_history, ('LTC/BTC', '1d'): ohlcv_history, - ('XRP/BTC', '1d'): ohlcv_history.iloc[[0]], } mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -837,16 +836,28 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o assert len(freqtrade.pairlists.whitelist) == 3 assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 - previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count freqtrade.pairlists.refresh_pairlist() assert len(freqtrade.pairlists.whitelist) == 3 # Call to XRP/BTC cached - assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count - # Move to next day - t.move_to("2021-09-02 01:00:00 +00:00") + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 2 + + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + ('XRP/BTC', '1d'): ohlcv_history.iloc[[0]], + } + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) freqtrade.pairlists.refresh_pairlist() assert len(freqtrade.pairlists.whitelist) == 3 - assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1 + + # Move to next day + t.move_to("2021-09-02 01:00:00 +00:00") + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1 # Move another day with fresh mocks (now the pair is old enough) t.move_to("2021-09-03 01:00:00 +00:00") From 4e2b1764b8af34f67305f213ac29dcff3a59da37 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Sep 2021 20:24:44 +0200 Subject: [PATCH 076/239] ccxt_compat_tests must run with dry-run=False --- tests/exchange/test_ccxt_compat.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 0f7963e36..d71dbe015 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -54,7 +54,9 @@ EXCHANGES = { def exchange_conf(): config = get_default_conf((Path(__file__).parent / "testdata").resolve()) config['exchange']['pair_whitelist'] = [] - # config['dry_run'] = False + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + config['dry_run'] = False return config From f7bae81d968a20e815d97d1dc72f86f0573edad2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Sep 2021 19:56:12 +0200 Subject: [PATCH 077/239] Dataframe should be copied after populate_indicator Without that, PerformanceWarnings can appear throughout hyperopt which are unnecessary and missleading for users closes #5408 --- freqtrade/strategy/interface.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 91963f223..00ad3faf0 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -777,10 +777,11 @@ class IStrategy(ABC, HyperStrategyMixin): Does not run advise_buy or advise_sell! Used by optimize operations only, not during dry / live runs. Using .copy() to get a fresh copy of the dataframe for every strategy run. + Also copy on output to avoid PerformanceWarnings pandas 1.3.0 started to show. Has positive effects on memory usage for whatever reason - also when using only one strategy. """ - return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair}) + return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair}).copy() for pair, pair_data in data.items()} def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: From 57ea0c322f8018992cb6c4303a43af29da310227 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Sep 2021 20:20:31 +0200 Subject: [PATCH 078/239] Rename indicator_space to buy_indicator_space --- docs/hyperopt.md | 2 +- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/optimize/hyperopt_auto.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index e69b761c4..09d43939a 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -677,7 +677,7 @@ If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace f These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the timeframe used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the timeframe used. -If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt file, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default. +If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default. Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps). diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 14b155546..d047b7311 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -241,7 +241,7 @@ class Hyperopt: if HyperoptTools.has_space(self.config, 'buy'): logger.debug("Hyperopt has 'buy' space") - self.buy_space = self.custom_hyperopt.indicator_space() + self.buy_space = self.custom_hyperopt.buy_indicator_space() if HyperoptTools.has_space(self.config, 'sell'): logger.debug("Hyperopt has 'sell' space") diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py index 1f11cec80..80fe090c2 100644 --- a/freqtrade/optimize/hyperopt_auto.py +++ b/freqtrade/optimize/hyperopt_auto.py @@ -56,7 +56,7 @@ class HyperOptAuto(IHyperOpt): else: _format_exception_message(category) - def indicator_space(self) -> List['Dimension']: + def buy_indicator_space(self) -> List['Dimension']: return self._get_indicator_space('buy') def sell_indicator_space(self) -> List['Dimension']: From 90ad1789323e2c3061110f34bd5d3c54d60b65d7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Sep 2021 21:04:25 +0200 Subject: [PATCH 079/239] Remove verbosity of edge --- freqtrade/plugins/pairlist/AgeFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index c43bd0c4c..5627d82ce 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -77,7 +77,7 @@ class AgeFilter(IPairList): if not needed_pairs: # Remove pairs that have been removed before return [p for p in pairlist if p not in self._symbolsCheckFailed] - logger.info(f"needed pairs {needed_pairs}") + since_days = -( self._max_days_listed if self._max_days_listed else self._min_days_listed ) - 1 From c0811ae8969799857e987ed7e93d1c1e78dd3a2f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Sep 2021 21:36:53 +0200 Subject: [PATCH 080/239] Add possibility to override estimator from within hyperopt --- docs/advanced-hyperopt.md | 32 ++++++++++++++++++++++++ freqtrade/optimize/hyperopt.py | 8 ++++-- freqtrade/optimize/hyperopt_auto.py | 5 +++- freqtrade/optimize/hyperopt_interface.py | 13 +++++++++- 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index f2f52b7dd..f5a52ff49 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -98,6 +98,38 @@ class MyAwesomeStrategy(IStrategy): !!! Note All overrides are optional and can be mixed/matched as necessary. +### Overriding Base estimator + +You can define your own estimator for Hyperopt by implementing `generate_estimator()` in the Hyperopt subclass. + +```python +class MyAwesomeStrategy(IStrategy): + class HyperOpt: + def generate_estimator(): + return "RF" + +``` + +Possible values are either one of "GP", "RF", "ET", "GBRT" (Details can be found in the [scikit-optimize documentation](https://scikit-optimize.github.io/)), or "an instance of a class that inherits from `RegressorMixin` (from sklearn) and where the `predict` method has an optional `return_std` argument, which returns `std(Y | x)` along with `E[Y | x]`". + +Some research will be necessary to find additional Regressors. + +Example for `ExtraTreesRegressor` ("ET") with additional parameters: + +```python +class MyAwesomeStrategy(IStrategy): + class HyperOpt: + def generate_estimator(): + from skopt.learning import ExtraTreesRegressor + # Corresponds to "ET" - but allows additional parameters. + return ExtraTreesRegressor(n_estimators=100) + +``` + +!!! Note + While custom estimators can be provided, it's up to you as User to do research on possible parameters and analyze / understand which ones should be used. + If you're unsure about this, best use one of the Defaults (`"ET"` has proven to be the most versatile) without further parameters. + ## Space options For the additional spaces, scikit-optimize (in combination with Freqtrade) provides the following space types: diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index d047b7311..56d11934a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -365,10 +365,14 @@ class Hyperopt: } def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer: + estimator = self.custom_hyperopt.generate_estimator() + logger.info(f"Using estimator {estimator}.") + # TODO: Impact of changing acq_optimizer to "sampling" is unclear + # (other than that it fails with other optimizers when using custom sklearn regressors) return Optimizer( dimensions, - base_estimator="ET", - acq_optimizer="auto", + base_estimator=estimator, + acq_optimizer="sampling", n_initial_points=INITIAL_POINTS, acq_optimizer_kwargs={'n_jobs': cpu_count}, random_state=self.random_state, diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py index 80fe090c2..c1c769c72 100644 --- a/freqtrade/optimize/hyperopt_auto.py +++ b/freqtrade/optimize/hyperopt_auto.py @@ -12,7 +12,7 @@ from freqtrade.exceptions import OperationalException with suppress(ImportError): from skopt.space import Dimension -from freqtrade.optimize.hyperopt_interface import IHyperOpt +from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt def _format_exception_message(space: str) -> str: @@ -79,3 +79,6 @@ class HyperOptAuto(IHyperOpt): def trailing_space(self) -> List['Dimension']: return self._get_func('trailing_space')() + + def generate_estimator(self) -> EstimatorType: + return self._get_func('generate_estimator')() diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 8fb40f557..53b4f087c 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -5,8 +5,9 @@ This module defines the interface to apply for hyperopt import logging import math from abc import ABC -from typing import Dict, List +from typing import Dict, List, Union +from sklearn.base import RegressorMixin from skopt.space import Categorical, Dimension, Integer from freqtrade.exchange import timeframe_to_minutes @@ -17,6 +18,8 @@ from freqtrade.strategy import IStrategy logger = logging.getLogger(__name__) +EstimatorType = Union[RegressorMixin, str] + class IHyperOpt(ABC): """ @@ -37,6 +40,14 @@ class IHyperOpt(ABC): IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED IHyperOpt.timeframe = str(config['timeframe']) + def generate_estimator(self) -> EstimatorType: + """ + Return base_estimator. + Can be any of "GP", "RF", "ET", "GBRT" or an instance of a class + inheriting from RegressorMixin (from sklearn). + """ + return 'ET' + def generate_roi_table(self, params: Dict) -> Dict[int, float]: """ Create a ROI table. From 994c3c3a4c5f36c02d249f4c13466b284c4991af Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Sep 2021 07:13:25 +0200 Subject: [PATCH 081/239] Add some errorhandling for custom estimator --- freqtrade/optimize/hyperopt.py | 14 ++++++++++---- tests/optimize/test_hyperopt.py | 4 ++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 56d11934a..9549b4054 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -45,7 +45,7 @@ progressbar.streams.wrap_stdout() logger = logging.getLogger(__name__) -INITIAL_POINTS = 30 +INITIAL_POINTS = 5 # Keep no more than SKOPT_MODEL_QUEUE_SIZE models # in the skopt model queue, to optimize memory consumption @@ -366,13 +366,19 @@ class Hyperopt: def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer: estimator = self.custom_hyperopt.generate_estimator() + + acq_optimizer = "sampling" + if isinstance(estimator, str): + if estimator not in ("GP", "RF", "ET", "GBRT"): + raise OperationalException(f"Estimator {estimator} not supported.") + else: + acq_optimizer = "auto" + logger.info(f"Using estimator {estimator}.") - # TODO: Impact of changing acq_optimizer to "sampling" is unclear - # (other than that it fails with other optimizers when using custom sklearn regressors) return Optimizer( dimensions, base_estimator=estimator, - acq_optimizer="sampling", + acq_optimizer=acq_optimizer, n_initial_points=INITIAL_POINTS, acq_optimizer_kwargs={'n_jobs': cpu_count}, random_state=self.random_state, diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index b34c3a916..e4ce29d44 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -884,6 +884,10 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: assert hyperopt.backtesting.strategy.buy_rsi.value != 35 assert hyperopt.backtesting.strategy.sell_rsi.value != 74 + hyperopt.custom_hyperopt.generate_estimator = lambda *args, **kwargs: 'ET1' + with pytest.raises(OperationalException, match="Estimator ET1 not supported."): + hyperopt.get_optimizer([], 2) + def test_SKDecimal(): space = SKDecimal(1, 2, decimals=2) From 457e738b4a049cbc522987612dc2e4bd91b272e9 Mon Sep 17 00:00:00 2001 From: Sergey Khliustin Date: Thu, 16 Sep 2021 14:48:02 +0300 Subject: [PATCH 082/239] Added days parameter to PerformanceFilter --- freqtrade/persistence/models.py | 28 ++++++++++++++++++- .../plugins/pairlist/PerformanceFilter.py | 15 ++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 8c8c1e0a9..91a26eba7 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -2,7 +2,7 @@ This module contains the class to persist trades into SQLite """ import logging -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from decimal import Decimal from typing import Any, Dict, List, Optional @@ -856,6 +856,32 @@ class Trade(_DECL_BASE, LocalTrade): for pair, profit, profit_abs, count in pair_rates ] + @staticmethod + def get_performance(days: int) -> List[Dict[str, Any]]: + """ + Returns List of dicts containing all Trades, including profit and trade count + NOTE: Not supported in Backtesting. + """ + start_date = datetime.today() - timedelta(days) + pair_rates = Trade.query.with_entities( + Trade.pair, + func.sum(Trade.close_profit).label('profit_sum'), + func.sum(Trade.close_profit_abs).label('profit_sum_abs'), + func.count(Trade.pair).label('count') + ).filter(Trade.is_open.is_(False) & (Trade.close_date >= start_date))\ + .group_by(Trade.pair) \ + .order_by(desc('profit_sum_abs')) \ + .all() + return [ + { + 'pair': pair, + 'profit': profit, + 'profit_abs': profit_abs, + 'count': count + } + for pair, profit, profit_abs, count in pair_rates + ] + @staticmethod def get_best_pair(start_date: datetime = datetime.fromtimestamp(0)): """ diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 46a289ae6..4d530fe88 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -2,11 +2,12 @@ Performance pair list filter """ import logging -from typing import Dict, List +from typing import Dict, List, Any import pandas as pd from freqtrade.persistence import Trade +from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList @@ -15,6 +16,13 @@ logger = logging.getLogger(__name__) class PerformanceFilter(IPairList): + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._days = pairlistconfig.get('days', 0) + @property def needstickers(self) -> bool: """ @@ -40,7 +48,10 @@ class PerformanceFilter(IPairList): """ # Get the trading performance for pairs from database try: - performance = pd.DataFrame(Trade.get_overall_performance()) + if self._days > 0: + performance = pd.DataFrame(Trade.get_performance(self._days)) + else: + performance = pd.DataFrame(Trade.get_overall_performance()) except AttributeError: # Performancefilter does not work in backtesting. self.log_once("PerformanceFilter is not available in this mode.", logger.warning) From 54ef36a4971d77467a10fa343857aec27a312e5a Mon Sep 17 00:00:00 2001 From: Sergey Khliustin Date: Fri, 17 Sep 2021 13:45:44 +0300 Subject: [PATCH 083/239] Updates after review to PerformanceFilter days param --- freqtrade/persistence/models.py | 34 ++++--------------- .../plugins/pairlist/PerformanceFilter.py | 5 +-- 2 files changed, 7 insertions(+), 32 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 91a26eba7..222be7169 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -832,43 +832,21 @@ class Trade(_DECL_BASE, LocalTrade): return total_open_stake_amount or 0 @staticmethod - def get_overall_performance() -> List[Dict[str, Any]]: + def get_overall_performance(days=None) -> List[Dict[str, Any]]: """ Returns List of dicts containing all Trades, including profit and trade count NOTE: Not supported in Backtesting. """ + filters = [Trade.is_open.is_(False)] + if days: + start_date = datetime.today() - timedelta(days) + filters.append((Trade.close_date >= start_date)) pair_rates = Trade.query.with_entities( Trade.pair, func.sum(Trade.close_profit).label('profit_sum'), func.sum(Trade.close_profit_abs).label('profit_sum_abs'), func.count(Trade.pair).label('count') - ).filter(Trade.is_open.is_(False))\ - .group_by(Trade.pair) \ - .order_by(desc('profit_sum_abs')) \ - .all() - return [ - { - 'pair': pair, - 'profit': profit, - 'profit_abs': profit_abs, - 'count': count - } - for pair, profit, profit_abs, count in pair_rates - ] - - @staticmethod - def get_performance(days: int) -> List[Dict[str, Any]]: - """ - Returns List of dicts containing all Trades, including profit and trade count - NOTE: Not supported in Backtesting. - """ - start_date = datetime.today() - timedelta(days) - pair_rates = Trade.query.with_entities( - Trade.pair, - func.sum(Trade.close_profit).label('profit_sum'), - func.sum(Trade.close_profit_abs).label('profit_sum_abs'), - func.count(Trade.pair).label('count') - ).filter(Trade.is_open.is_(False) & (Trade.close_date >= start_date))\ + ).filter(filters)\ .group_by(Trade.pair) \ .order_by(desc('profit_sum_abs')) \ .all() diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 4d530fe88..920b884a0 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -48,10 +48,7 @@ class PerformanceFilter(IPairList): """ # Get the trading performance for pairs from database try: - if self._days > 0: - performance = pd.DataFrame(Trade.get_performance(self._days)) - else: - performance = pd.DataFrame(Trade.get_overall_performance()) + performance = pd.DataFrame(Trade.get_overall_performance(self._days)) except AttributeError: # Performancefilter does not work in backtesting. self.log_once("PerformanceFilter is not available in this mode.", logger.warning) From 982deeedf04cb1be9c5ae7b1ec3598a9d09a4b9a Mon Sep 17 00:00:00 2001 From: sergeykhliustin <51409210+sergeykhliustin@users.noreply.github.com> Date: Fri, 17 Sep 2021 18:23:13 +0300 Subject: [PATCH 084/239] Update freqtrade/persistence/models.py Co-authored-by: Matthias --- freqtrade/persistence/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 222be7169..a2e30d58c 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -846,7 +846,7 @@ class Trade(_DECL_BASE, LocalTrade): func.sum(Trade.close_profit).label('profit_sum'), func.sum(Trade.close_profit_abs).label('profit_sum_abs'), func.count(Trade.pair).label('count') - ).filter(filters)\ + ).filter(*filters)\ .group_by(Trade.pair) \ .order_by(desc('profit_sum_abs')) \ .all() From 3a98fb72a491536cef7b601573c3ca45f64e8fc5 Mon Sep 17 00:00:00 2001 From: raphael Date: Fri, 17 Sep 2021 11:42:33 -0400 Subject: [PATCH 085/239] Update prepare_trials_columns() return type Was returning str, updated to pd.DataFrame --- freqtrade/optimize/hyperopt_tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index b2e024f65..2fcab0b0f 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any, Dict, Iterator, List, Optional, Tuple import numpy as np +import pandas as pd import rapidjson import tabulate from colorama import Fore, Style @@ -298,7 +299,7 @@ class HyperoptTools(): f"Objective: {results['loss']:.5f}") @staticmethod - def prepare_trials_columns(trials, legacy_mode: bool, has_drawdown: bool) -> str: + def prepare_trials_columns(trials, legacy_mode: bool, has_drawdown: bool) -> pd.DataFrame: trials['Best'] = '' From 5fc993231a2d138e9ff994f1d4548888551dd73c Mon Sep 17 00:00:00 2001 From: raphael Date: Fri, 17 Sep 2021 11:51:55 -0400 Subject: [PATCH 086/239] Update HyperoptTools.export_csv_file usage --- freqtrade/commands/hyperopt_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 089529d15..614c4b3f5 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -53,7 +53,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: if epochs and export_csv: HyperoptTools.export_csv_file( - config, epochs, total_epochs, not config.get('hyperopt_list_best', False), export_csv + config, epochs, export_csv ) From 124e97f3b9a7006cfcf2337255f10c8bcc3e0038 Mon Sep 17 00:00:00 2001 From: raphael Date: Fri, 17 Sep 2021 11:57:36 -0400 Subject: [PATCH 087/239] Remove ununsed variables from export_csv_file --- freqtrade/optimize/hyperopt_tools.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index b2e024f65..6a5c31d56 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -435,8 +435,7 @@ class HyperoptTools(): return table @staticmethod - def export_csv_file(config: dict, results: list, total_epochs: int, highlight_best: bool, - csv_file: str) -> None: + def export_csv_file(config: dict, results: list, csv_file: str) -> None: """ Log result to csv-file """ From fb6beb90e87e75d2997651324daaf0c5954e661a Mon Sep 17 00:00:00 2001 From: Ottavio Miele Date: Fri, 17 Sep 2021 18:03:54 +0200 Subject: [PATCH 088/239] Include Raspberry Pi armv6 (0, 0W,...) --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index 217500569..aee7c80b5 100755 --- a/setup.sh +++ b/setup.sh @@ -62,7 +62,7 @@ function updateenv() { then REQUIREMENTS_PLOT="-r requirements-plot.txt" fi - if [ "${SYS_ARCH}" == "armv7l" ]; then + if [ "${SYS_ARCH}" == "armv7l" ] || [ "${SYS_ARCH}" == "armv6l" ]; then echo "Detected Raspberry, installing cython, skipping hyperopt installation." ${PYTHON} -m pip install --upgrade cython else From 9525a5b96cf779dfa80abdae355e2d5c9382f505 Mon Sep 17 00:00:00 2001 From: raphael Date: Fri, 17 Sep 2021 14:10:37 -0400 Subject: [PATCH 089/239] Add type to "trials" parameter --- freqtrade/optimize/hyperopt_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 2fcab0b0f..1874b2949 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -299,7 +299,7 @@ class HyperoptTools(): f"Objective: {results['loss']:.5f}") @staticmethod - def prepare_trials_columns(trials, legacy_mode: bool, has_drawdown: bool) -> pd.DataFrame: + def prepare_trials_columns(trials: pd.DataFrame, legacy_mode: bool, has_drawdown: bool) -> pd.DataFrame: trials['Best'] = '' From e715f2a253afe2ef4332166c0357595976dd7dfc Mon Sep 17 00:00:00 2001 From: raphael Date: Fri, 17 Sep 2021 14:23:26 -0400 Subject: [PATCH 090/239] Update formatting Line 302 was too long --- freqtrade/optimize/hyperopt_tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 1874b2949..ac92ad682 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -299,7 +299,8 @@ class HyperoptTools(): f"Objective: {results['loss']:.5f}") @staticmethod - def prepare_trials_columns(trials: pd.DataFrame, legacy_mode: bool, has_drawdown: bool) -> pd.DataFrame: + def prepare_trials_columns(trials: pd.DataFrame, legacy_mode: bool, + has_drawdown: bool) -> pd.DataFrame: trials['Best'] = '' From 4b2c1a9b8e0e487a60454125e8916026b8443c5c Mon Sep 17 00:00:00 2001 From: raphael Date: Fri, 17 Sep 2021 14:39:15 -0400 Subject: [PATCH 091/239] Remove trailing whitespace --- freqtrade/optimize/hyperopt_tools.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index ac92ad682..7a4ce5f23 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -299,9 +299,8 @@ class HyperoptTools(): f"Objective: {results['loss']:.5f}") @staticmethod - def prepare_trials_columns(trials: pd.DataFrame, legacy_mode: bool, + def prepare_trials_columns(trials: pd.DataFrame, legacy_mode: bool, has_drawdown: bool) -> pd.DataFrame: - trials['Best'] = '' if 'results_metrics.winsdrawslosses' not in trials.columns: From d7395e873be5560932752e00200c70fc2cd169c3 Mon Sep 17 00:00:00 2001 From: Sergey Khliustin Date: Fri, 17 Sep 2021 22:05:57 +0300 Subject: [PATCH 092/239] Removed unused OperationalException --- freqtrade/plugins/pairlist/PerformanceFilter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 920b884a0..5750b721c 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -7,7 +7,6 @@ from typing import Dict, List, Any import pandas as pd from freqtrade.persistence import Trade -from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList From 12c12d42df5e0f838aad1ef971ca8530345d2d00 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Sep 2021 08:26:48 +0200 Subject: [PATCH 093/239] Add documentation for days parameter in PerformanceFilter --- docs/includes/pairlists.md | 14 ++++++++++++++ freqtrade/persistence/models.py | 4 ++-- freqtrade/plugins/pairlist/PerformanceFilter.py | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 69e12d5dc..71165f93b 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -165,6 +165,7 @@ Example to remove the first 10 pairs from the pairlist: ```json "pairlists": [ + // ... { "method": "OffsetFilter", "offset": 10 @@ -190,6 +191,19 @@ Sorts pairs by past trade performance, as follows: Trade count is used as a tie breaker. +You can use the `days` parameter to only consider performance of the past X days. +Not defining this parameter (or setting it to 0) will use all-time performance. + +```json +"pairlists": [ + // ... + { + "method": "PerformanceFilter", + "days": 10 + } +], +``` + !!! Note `PerformanceFilter` does not support backtesting mode. diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index a2e30d58c..f5f4e3a26 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -2,7 +2,7 @@ This module contains the class to persist trades into SQLite """ import logging -from datetime import datetime, timezone, timedelta +from datetime import datetime, timedelta, timezone from decimal import Decimal from typing import Any, Dict, List, Optional @@ -840,7 +840,7 @@ class Trade(_DECL_BASE, LocalTrade): filters = [Trade.is_open.is_(False)] if days: start_date = datetime.today() - timedelta(days) - filters.append((Trade.close_date >= start_date)) + filters.append(Trade.close_date >= start_date) pair_rates = Trade.query.with_entities( Trade.pair, func.sum(Trade.close_profit).label('profit_sum'), diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 5750b721c..ee443b0fe 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -2,7 +2,7 @@ Performance pair list filter """ import logging -from typing import Dict, List, Any +from typing import Any, Dict, List import pandas as pd From 564e0b9a1ac89c19f2a9c1e94c04cafd4fd96afe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Sep 2021 08:36:06 +0200 Subject: [PATCH 094/239] Switch performanceFilter to use Minutes lookback resolution closes #5060 --- docs/includes/pairlists.md | 4 ++-- freqtrade/persistence/models.py | 6 +++--- freqtrade/plugins/pairlist/PerformanceFilter.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 71165f93b..b612a4ddf 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -191,7 +191,7 @@ Sorts pairs by past trade performance, as follows: Trade count is used as a tie breaker. -You can use the `days` parameter to only consider performance of the past X days. +You can use the `minutes` parameter to only consider performance of the past X minutes (rolling window). Not defining this parameter (or setting it to 0) will use all-time performance. ```json @@ -199,7 +199,7 @@ Not defining this parameter (or setting it to 0) will use all-time performance. // ... { "method": "PerformanceFilter", - "days": 10 + "minutes": 1440 // rolling 24h } ], ``` diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index f5f4e3a26..bc5ef961a 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -832,14 +832,14 @@ class Trade(_DECL_BASE, LocalTrade): return total_open_stake_amount or 0 @staticmethod - def get_overall_performance(days=None) -> List[Dict[str, Any]]: + def get_overall_performance(minutes=None) -> List[Dict[str, Any]]: """ Returns List of dicts containing all Trades, including profit and trade count NOTE: Not supported in Backtesting. """ filters = [Trade.is_open.is_(False)] - if days: - start_date = datetime.today() - timedelta(days) + if minutes: + start_date = datetime.now(timezone.utc) - timedelta(minutes=minutes) filters.append(Trade.close_date >= start_date) pair_rates = Trade.query.with_entities( Trade.pair, diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index ee443b0fe..301ee57ab 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -20,7 +20,7 @@ class PerformanceFilter(IPairList): pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - self._days = pairlistconfig.get('days', 0) + self._minutes = pairlistconfig.get('minutes', 0) @property def needstickers(self) -> bool: @@ -47,7 +47,7 @@ class PerformanceFilter(IPairList): """ # Get the trading performance for pairs from database try: - performance = pd.DataFrame(Trade.get_overall_performance(self._days)) + performance = pd.DataFrame(Trade.get_overall_performance(self._minutes)) except AttributeError: # Performancefilter does not work in backtesting. self.log_once("PerformanceFilter is not available in this mode.", logger.warning) From 56fb25c5e521415c9880e47c4938bfe357550bec Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Sep 2021 09:10:37 +0200 Subject: [PATCH 095/239] Add test for PerformanceFilter lookback --- tests/plugins/test_pairlist.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 34770c03d..1ce8d172c 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -12,7 +12,8 @@ from freqtrade.persistence import Trade from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver -from tests.conftest import get_patched_exchange, get_patched_freqtradebot, log_has, log_has_re +from tests.conftest import (create_mock_trades, get_patched_exchange, get_patched_freqtradebot, + log_has, log_has_re) @pytest.fixture(scope="function") @@ -663,6 +664,31 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None: assert log_has("PerformanceFilter is not available in this mode.", caplog) +@pytest.mark.usefixtures("init_persistence") +def test_PerformanceFilter_lookback(mocker, whitelist_conf, fee) -> None: + whitelist_conf['exchange']['pair_whitelist'].append('XRP/BTC') + whitelist_conf['pairlists'] = [ + {"method": "StaticPairList"}, + {"method": "PerformanceFilter", "minutes": 60} + ] + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + exchange = get_patched_exchange(mocker, whitelist_conf) + pm = PairListManager(exchange, whitelist_conf) + pm.refresh_pairlist() + + assert pm.whitelist == ['ETH/BTC', 'TKN/BTC', 'XRP/BTC'] + + with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: + create_mock_trades(fee) + pm.refresh_pairlist() + assert pm.whitelist == ['XRP/BTC', 'ETH/BTC', 'TKN/BTC'] + + # Move to "outside" of lookback window, so original sorting is restored. + t.move_to("2021-09-01 07:00:00 +00:00") + pm.refresh_pairlist() + assert pm.whitelist == ['ETH/BTC', 'TKN/BTC', 'XRP/BTC'] + + def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}] From d84ef34740a65df4922a13b8e8e523167637ae2e Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sat, 17 Jul 2021 18:09:08 +0300 Subject: [PATCH 096/239] A helper to calculate stoploss value from absolute price. --- docs/strategy-advanced.md | 6 ++++ docs/strategy-customization.md | 41 +++++++++++++++++++++++++++ freqtrade/strategy/__init__.py | 3 +- freqtrade/strategy/strategy_helper.py | 11 +++++++ 4 files changed, 60 insertions(+), 1 deletion(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 4409af6ea..2b9517f3b 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -288,6 +288,12 @@ Stoploss values returned from `custom_stoploss()` always specify a percentage re The helper function [`stoploss_from_open()`](strategy-customization.md#stoploss_from_open) can be used to convert from an open price relative stop, to a current price relative stop which can be returned from `custom_stoploss()`. +### Calculating stoploss percentage from absolute price + +Stoploss values returned from `custom_stoploss()` always specify a percentage relative to `current_rate`. In order to set a stoploss at specified absolute price level, we need to use `stop_rate` to calculate what percentage relative to the `current_rate` will give you the same result as if the percentage was specified from the open price. + +The helper function [`stoploss_from_absolute()`](strategy-customization.md#stoploss_from_absolute) can be used to convert from an absolute price, to a current price relative stop which can be returned from `custom_stoploss()`. + #### Stepped stoploss Instead of continuously trailing behind the current price, this example sets fixed stoploss price levels based on the current profit. diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index cfea60d22..1f8116deb 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -639,6 +639,47 @@ Stoploss values returned from `custom_stoploss` must specify a percentage relati Full examples can be found in the [Custom stoploss](strategy-advanced.md#custom-stoploss) section of the Documentation. +!!! Note + Providing invalid input to `stoploss_from_open()` may produce "CustomStoploss function did not return valid stoploss" warnings. + This may happen if `current_profit` parameter is below specified `open_relative_stop`. Such situations may arise when closing trade + is blocked by `confirm_trade_exit()` method. Warnings can be solved by never blocking stop loss sells by checking `sell_reason` in + `confirm_trade_exit()`, or by using `return stoploss_from_open(...) or 1` idiom, which will request to not change stop loss when + `current_profit < open_relative_stop`. + +### *stoploss_from_absolute()* + +In some situations it may be confusing to deal with stops relative to current rate. Instead, you may define a stoploss level using an absolute price. + +??? Example "Returning a stoploss using absolute price from the custom stoploss function" + + Say the open price was $100, and `current_price` is $121 (`current_profit` will be `0.21`). + + If we want a stop price at $107 price we can call `stoploss_from_absolute(107, current_rate)` which will return `0.1157024793`. 11.57% below $121 is $107, which is the same as 7% above $100. + + ``` python + + from datetime import datetime + from freqtrade.persistence import Trade + from freqtrade.strategy import IStrategy, stoploss_from_open + + class AwesomeStrategy(IStrategy): + + # ... populate_* methods + + use_custom_stoploss = True + + def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, + current_rate: float, current_profit: float, **kwargs) -> float: + + # once the profit has risen above 10%, keep the stoploss at 7% above the open price + if current_profit > 0.10: + return stoploss_from_absolute(trade.open_rate * 1.07, current_rate) + + return 1 + + ``` + + Full examples can be found in the [Custom stoploss](strategy-advanced.md#custom-stoploss) section of the Documentation. ## Additional data (Wallets) diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index be655fc33..703cdabc1 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -4,4 +4,5 @@ from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_msecs, timefr from freqtrade.strategy.hyper import (BooleanParameter, CategoricalParameter, DecimalParameter, IntParameter, RealParameter) from freqtrade.strategy.interface import IStrategy -from freqtrade.strategy.strategy_helper import merge_informative_pair, stoploss_from_open +from freqtrade.strategy.strategy_helper import (merge_informative_pair, + stoploss_from_absolute, stoploss_from_open) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index e089ebf31..32f7a9886 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -83,3 +83,14 @@ def stoploss_from_open(open_relative_stop: float, current_profit: float) -> floa # negative stoploss values indicate the requested stop price is higher than the current price return max(stoploss, 0.0) + + +def stoploss_from_absolute(stop_rate: float, current_rate: float) -> float: + """ + Given current price and desired stop price, return a stop loss value that is relative to current + price. + :param stop_rate: Stop loss price. + :param current_rate: Current asset price. + :return: Positive stop loss value relative to current price + """ + return 1 - (stop_rate / current_rate) From 1fdb656334208f57916fe4539ab83e9363d7b984 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sat, 17 Jul 2021 19:19:49 +0300 Subject: [PATCH 097/239] Add a decorator which can be used to declare populate_indicators() functions for informative pairs. --- docs/strategy-customization.md | 84 +++++++++- freqtrade/edge/edge_positioning.py | 2 +- freqtrade/freqtradebot.py | 2 +- freqtrade/strategy/__init__.py | 2 +- freqtrade/strategy/interface.py | 53 ++++++- freqtrade/strategy/strategy_helper.py | 148 +++++++++++++++++- tests/rpc/test_rpc_apiserver.py | 1 + .../strats/informative_decorator_strategy.py | 75 +++++++++ tests/strategy/test_interface.py | 2 +- tests/strategy/test_strategy_helpers.py | 55 +++++++ tests/strategy/test_strategy_loading.py | 6 +- 11 files changed, 414 insertions(+), 16 deletions(-) create mode 100644 tests/strategy/strats/informative_decorator_strategy.py diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 1f8116deb..526c111c5 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -679,7 +679,89 @@ In some situations it may be confusing to deal with stops relative to current ra ``` - Full examples can be found in the [Custom stoploss](strategy-advanced.md#custom-stoploss) section of the Documentation. +### *@informative()* + +In most common case it is possible to easily define informative pairs by using a decorator. All decorated `populate_indicators_*` methods run in isolation, +not having access to data from other informative pairs, in the end all informative dataframes are merged and passed to main `populate_indicators()` method. +When hyperopting, please follow instructions of [optimizing an indicator parameter](hyperopt.md#optimizing-an-indicator-parameter). + +??? Example "Fast and easy way to define informative pairs" + + Most of the time we do not need power and flexibility offered by `merge_informative_pair()`, therefore we can use a decorator to quickly define informative pairs. + + ``` python + + from datetime import datetime + from freqtrade.persistence import Trade + from freqtrade.strategy import IStrategy, informative + + class AwesomeStrategy(IStrategy): + + # This method is not required. + # def informative_pairs(self): ... + + # Define informative upper timeframe for each pair. Decorators can be stacked on same + # method. Available in populate_indicators as 'rsi_30m' and 'rsi_1h'. + @informative('30m') + @informative('1h') + def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + return dataframe + + # Define BTC/STAKE informative pair. Available in populate_indicators and other methods as + # 'btc_rsi_1h'. Current stake currency should be specified as {stake} format variable + # instead of hardcoding actual stake currency. Available in populate_indicators and other + # methods as 'btc_rsi_1h'. + @informative('1h', 'BTC/{stake}') + def populate_indicators_btc_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + return dataframe + + # Define BTC/ETH informative pair. You must specify quote currency if it is different from + # stake currency. Available in populate_indicators and other methods as 'eth_btc_rsi_1h'. + @informative('1h', 'ETH/BTC') + def populate_indicators_eth_btc_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + return dataframe + + # Define BTC/STAKE informative pair. A custom formatter may be specified for formatting + # column names. Format string supports these format variables: + # * {asset} - full name of the asset, for example 'BTC/USDT'. + # * {base} - base currency in lower case, for example 'eth'. + # * {BASE} - same as {base}, except in upper case. + # * {quote} - quote currency in lower case, for example 'usdt'. + # * {QUOTE} - same as {quote}, except in upper case. + # * {column} - name of dataframe column. + # * {timeframe} - timeframe of informative dataframe. + # A callable `fmt(**kwargs) -> str` may be specified, to implement custom formatting. + # Available in populate_indicators and other methods as 'rsi_upper'. + @informative('1h', 'BTC/{stake}', '{name}') + def populate_indicators_btc_1h_2(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi_upper'] = ta.RSI(dataframe, timeperiod=14) + return dataframe + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + # Strategy timeframe indicators for current pair. + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + # Informative pairs are available in this method. + dataframe['rsi_less'] = dataframe['rsi'] < dataframe['rsi_1h'] + return dataframe + + ``` + + See docstring of `@informative()` decorator for more information. + +!!! Note + Do not use `@informative` decorator if you need to use data of one informative pair when generating another informative pair. Instead, define informative pairs + manually as described [in the DataProvider section](#complete-data-provider-sample). + +!!! Warning + Methods tagged with `@informative()` decorator must always have unique names! Re-using same name (for example when copy-pasting already defined informative method) + will overwrite previously defined method and not produce any errors due to limitations of Python programming language. In such cases you will find that indicators + created in earlier-defined methods are not available in the dataframe. Carefully review method names and make sure they are unique! + +!!! Warning + When using a legacy hyperopt implementation informative pairs defined with a decorator will not be executed. Please update your strategy if necessary. ## Additional data (Wallets) diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index f12b1b37d..1950f0d08 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -119,7 +119,7 @@ class Edge: ) # Download informative pairs too res = defaultdict(list) - for p, t in self.strategy.informative_pairs(): + for p, t in self.strategy.gather_informative_pairs(): res[t].append(p) for timeframe, inf_pairs in res.items(): timerange_startup = deepcopy(self._timerange) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7f668273c..bdc438c9a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -160,7 +160,7 @@ class FreqtradeBot(LoggingMixin): # Refreshing candles self.dataprovider.refresh(self.pairlists.create_pair_list(self.active_pair_whitelist), - self.strategy.informative_pairs()) + self.strategy.gather_informative_pairs()) strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)() diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index 703cdabc1..a7de34916 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -4,5 +4,5 @@ from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_msecs, timefr from freqtrade.strategy.hyper import (BooleanParameter, CategoricalParameter, DecimalParameter, IntParameter, RealParameter) from freqtrade.strategy.interface import IStrategy -from freqtrade.strategy.strategy_helper import (merge_informative_pair, +from freqtrade.strategy.strategy_helper import (informative, merge_informative_pair, stoploss_from_absolute, stoploss_from_open) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 00ad3faf0..8e8b8b404 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -6,7 +6,7 @@ import logging import warnings from abc import ABC, abstractmethod from datetime import datetime, timedelta, timezone -from typing import Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union import arrow from pandas import DataFrame @@ -19,6 +19,8 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.exchange.exchange import timeframe_to_next_date from freqtrade.persistence import PairLocks, Trade from freqtrade.strategy.hyper import HyperStrategyMixin +from freqtrade.strategy.strategy_helper import (InformativeData, _create_and_merge_informative_pair, + _format_pair_name) from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets @@ -134,6 +136,39 @@ class IStrategy(ABC, HyperStrategyMixin): self._last_candle_seen_per_pair: Dict[str, datetime] = {} super().__init__(config) + # Gather informative pairs from @informative-decorated methods. + self._ft_informative: Dict[ + Tuple[str, str], Tuple[InformativeData, + Callable[[Any, DataFrame, dict], DataFrame]]] = {} + for attr_name in dir(self.__class__): + cls_method = getattr(self.__class__, attr_name) + if not callable(cls_method): + continue + ft_informative = getattr(cls_method, '_ft_informative', []) + if not isinstance(ft_informative, list): + # Type check is required because mocker would return a mock object that evaluates to + # True, confusing this code. + continue + for informative_data in ft_informative: + asset = informative_data.asset + timeframe = informative_data.timeframe + if asset: + pair = _format_pair_name(self.config, asset) + if (pair, timeframe) in self._ft_informative: + raise OperationalException(f'Informative pair {pair} {timeframe} can not ' + f'be defined more than once!') + self._ft_informative[(pair, timeframe)] = (informative_data, cls_method) + elif self.dp is not None: + for pair in self.dp.current_whitelist(): + if (pair, timeframe) in self._ft_informative: + raise OperationalException(f'Informative pair {pair} {timeframe} can ' + f'not be defined more than once!') + self._ft_informative[(pair, timeframe)] = (informative_data, cls_method) + + def _format_pair(self, pair: str) -> str: + return pair.format(stake_currency=self.config['stake_currency'], + stake=self.config['stake_currency']).upper() + @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ @@ -377,6 +412,14 @@ class IStrategy(ABC, HyperStrategyMixin): # END - Intended to be overridden by strategy ### + def gather_informative_pairs(self) -> ListPairsWithTimeframes: + """ + Internal method which gathers all informative pairs (user or automatically defined). + """ + informative_pairs = self.informative_pairs() + informative_pairs += list(self._ft_informative.keys()) + return list(set(informative_pairs)) + def get_strategy_name(self) -> str: """ Returns strategy class name @@ -793,6 +836,14 @@ class IStrategy(ABC, HyperStrategyMixin): :return: a Dataframe with all mandatory indicators for the strategies """ logger.debug(f"Populating indicators for pair {metadata.get('pair')}.") + + # call populate_indicators_Nm() which were tagged with @informative decorator. + for (pair, timeframe), (informative_data, populate_fn) in self._ft_informative.items(): + if not informative_data.asset and pair != metadata['pair']: + continue + dataframe = _create_and_merge_informative_pair( + self, dataframe, metadata, informative_data, populate_fn) + if self._populate_fun_len == 2: warnings.warn("deprecated - check out the Sample strategy to see " "the current function headers!", DeprecationWarning) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index 32f7a9886..aa828d330 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -1,10 +1,24 @@ -import pandas as pd +from typing import Any, Callable, NamedTuple, Optional, Union +import pandas as pd +from mypy_extensions import KwArg +from pandas import DataFrame + +from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes +class InformativeData(NamedTuple): + asset: Optional[str] + timeframe: str + fmt: Union[str, Callable[[KwArg(str)], str], None] + ffill: bool + + def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, - timeframe: str, timeframe_inf: str, ffill: bool = True) -> pd.DataFrame: + timeframe: str, timeframe_inf: str, ffill: bool = True, + append_timeframe: bool = True, + date_column: str = 'date') -> pd.DataFrame: """ Correctly merge informative samples to the original dataframe, avoiding lookahead bias. @@ -24,6 +38,8 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, :param timeframe: Timeframe of the original pair sample. :param timeframe_inf: Timeframe of the informative pair sample. :param ffill: Forwardfill missing values - optional but usually required + :param append_timeframe: Rename columns by appending timeframe. + :param date_column: A custom date column name. :return: Merged dataframe :raise: ValueError if the secondary timeframe is shorter than the dataframe timeframe """ @@ -32,25 +48,29 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, minutes = timeframe_to_minutes(timeframe) if minutes == minutes_inf: # No need to forwardshift if the timeframes are identical - informative['date_merge'] = informative["date"] + informative['date_merge'] = informative[date_column] elif minutes < minutes_inf: # Subtract "small" timeframe so merging is not delayed by 1 small candle # Detailed explanation in https://github.com/freqtrade/freqtrade/issues/4073 informative['date_merge'] = ( - informative["date"] + pd.to_timedelta(minutes_inf, 'm') - pd.to_timedelta(minutes, 'm') + informative[date_column] + pd.to_timedelta(minutes_inf, 'm') - + pd.to_timedelta(minutes, 'm') ) else: raise ValueError("Tried to merge a faster timeframe to a slower timeframe." "This would create new rows, and can throw off your regular indicators.") # Rename columns to be unique - informative.columns = [f"{col}_{timeframe_inf}" for col in informative.columns] + date_merge = 'date_merge' + if append_timeframe: + date_merge = f'date_merge_{timeframe_inf}' + informative.columns = [f"{col}_{timeframe_inf}" for col in informative.columns] # Combine the 2 dataframes # all indicators on the informative sample MUST be calculated before this point dataframe = pd.merge(dataframe, informative, left_on='date', - right_on=f'date_merge_{timeframe_inf}', how='left') - dataframe = dataframe.drop(f'date_merge_{timeframe_inf}', axis=1) + right_on=date_merge, how='left') + dataframe = dataframe.drop(date_merge, axis=1) if ffill: dataframe = dataframe.ffill() @@ -94,3 +114,117 @@ def stoploss_from_absolute(stop_rate: float, current_rate: float) -> float: :return: Positive stop loss value relative to current price """ return 1 - (stop_rate / current_rate) + + +def informative(timeframe: str, asset: str = '', + fmt: Optional[Union[str, Callable[[KwArg(str)], str]]] = None, + ffill: bool = True) -> Callable[[Callable[[Any, DataFrame, dict], DataFrame]], + Callable[[Any, DataFrame, dict], DataFrame]]: + """ + A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to + define informative indicators. + + Example usage: + + @informative('1h') + def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + return dataframe + + :param timeframe: Informative timeframe. Must always be higher than strategy timeframe. + :param asset: Informative asset, for example BTC, BTC/USDT, ETH/BTC. Do not specify to use + current pair. + :param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not + specified, defaults to {asset}_{name}_{timeframe} if asset is specified, or {name}_{timeframe} + otherwise. + * {asset}: name of informative asset, provided in lower-case, with / replaced with _. Stake + currency is not included in this string. + * {name}: user-specified dataframe column name. + * {timeframe}: informative timeframe. + :param ffill: ffill dataframe after mering informative pair. + """ + _asset = asset + _timeframe = timeframe + _fmt = fmt + _ffill = ffill + + def decorator(fn: Callable[[Any, DataFrame, dict], DataFrame]): + informative_pairs = getattr(fn, '_ft_informative', []) + informative_pairs.append(InformativeData(_asset, _timeframe, _fmt, _ffill)) + setattr(fn, '_ft_informative', informative_pairs) + return fn + return decorator + + +def _format_pair_name(config, pair: str) -> str: + return pair.format(stake_currency=config['stake_currency'], + stake=config['stake_currency']).upper() + + +def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, + metadata: dict, informative_data: InformativeData, + populate_indicators: Callable[[Any, DataFrame, dict], + DataFrame]): + asset = informative_data.asset or '' + timeframe = informative_data.timeframe + fmt = informative_data.fmt + ffill = informative_data.ffill + config = strategy.config + dp = strategy.dp + + if asset: + # Insert stake currency if needed. + asset = _format_pair_name(config, asset) + else: + # Not specifying an asset will define informative dataframe for current pair. + asset = metadata['pair'] + + if '/' in asset: + base, quote = asset.split('/') + else: + # When futures are supported this may need reevaluation. + # base, quote = asset, None + raise OperationalException('Not implemented.') + + # Default format. This optimizes for the common case: informative pairs using same stake + # currency. When quote currency matches stake currency, column name will omit base currency. + # This allows easily reconfiguring strategy to use different base currency. In a rare case + # where it is desired to keep quote currency in column name at all times user should specify + # fmt='{base}_{quote}_{column}_{timeframe}' format or similar. + if not fmt: + fmt = '{column}_{timeframe}' # Informatives of current pair + if asset != metadata['pair']: + if quote == config['stake_currency']: + fmt = '{base}_' + fmt # Informatives of other pair + else: + fmt = '{base}_{quote}_' + fmt # Informatives of different quote currency + + inf_metadata = {'pair': asset, 'timeframe': timeframe} + inf_dataframe = dp.get_pair_dataframe(asset, timeframe) + inf_dataframe = populate_indicators(strategy, inf_dataframe, inf_metadata) + + formatter: Any = None + if callable(fmt): + formatter = fmt # A custom user-specified formatter function. + else: + formatter = fmt.format # A default string formatter. + + fmt_args = { + 'BASE': base.upper(), + 'QUOTE': quote.upper(), + 'base': base.lower(), + 'quote': quote.lower(), + 'asset': asset, + 'timeframe': timeframe, + } + inf_dataframe.rename(columns=lambda column: formatter(column=column, **fmt_args), + inplace=True) + + date_column = formatter(column='date', **fmt_args) + if date_column in dataframe.columns: + raise OperationalException(f'Duplicate column name {date_column} exists in ' + f'dataframe! Ensure column names are unique!') + dataframe = merge_informative_pair(dataframe, inf_dataframe, strategy.timeframe, timeframe, + ffill=ffill, append_timeframe=False, + date_column=date_column) + return dataframe diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 2852486ed..43eb70938 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1218,6 +1218,7 @@ def test_api_strategies(botclient): assert_response(rc) assert rc.json() == {'strategies': [ 'HyperoptableStrategy', + 'InformativeDecoratorTest', 'StrategyTestV2', 'TestStrategyLegacyV1' ]} diff --git a/tests/strategy/strats/informative_decorator_strategy.py b/tests/strategy/strats/informative_decorator_strategy.py new file mode 100644 index 000000000..a32ad79e8 --- /dev/null +++ b/tests/strategy/strats/informative_decorator_strategy.py @@ -0,0 +1,75 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +from pandas import DataFrame + +from freqtrade.strategy import informative, merge_informative_pair +from freqtrade.strategy.interface import IStrategy + + +class InformativeDecoratorTest(IStrategy): + """ + Strategy used by tests freqtrade bot. + Please do not modify this strategy, it's intended for internal use only. + Please look at the SampleStrategy in the user_data/strategy directory + or strategy repository https://github.com/freqtrade/freqtrade-strategies + for samples and inspiration. + """ + INTERFACE_VERSION = 2 + stoploss = -0.10 + timeframe = '5m' + startup_candle_count: int = 20 + + def informative_pairs(self): + return [('BTC/USDT', '5m')] + + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['buy'] = 0 + return dataframe + + def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['sell'] = 0 + return dataframe + + # Decorator stacking test. + @informative('30m') + @informative('1h') + def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = 14 + return dataframe + + # Simple informative test. + @informative('1h', 'BTC/{stake}') + def populate_indicators_btc_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = 14 + return dataframe + + # Quote currency different from stake currency test. + @informative('1h', 'ETH/BTC') + def populate_indicators_eth_btc_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = 14 + return dataframe + + # Formatting test. + @informative('30m', 'BTC/{stake}', '{column}_{BASE}_{QUOTE}_{base}_{quote}_{asset}_{timeframe}') + def populate_indicators_btc_1h_2(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = 14 + return dataframe + + # Custom formatter test + @informative('30m', 'ETH/{stake}', fmt=lambda column, **kwargs: column + '_from_callable') + def populate_indicators_eth_30m(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = 14 + return dataframe + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + # Strategy timeframe indicators for current pair. + dataframe['rsi'] = 14 + # Informative pairs are available in this method. + dataframe['rsi_less'] = dataframe['rsi'] < dataframe['rsi_1h'] + + # Mixing manual informative pairs with decorators. + informative = self.dp.get_pair_dataframe('BTC/USDT', '5m') + informative['rsi'] = 14 + dataframe = merge_informative_pair(dataframe, informative, self.timeframe, '5m', ffill=True) + + return dataframe diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 250dcf63d..dcb9e3e64 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -607,7 +607,7 @@ def test_is_informative_pairs_callback(default_conf): strategy = StrategyResolver.load_strategy(default_conf) # Should return empty # Uses fallback to base implementation - assert [] == strategy.informative_pairs() + assert [] == strategy.gather_informative_pairs() @pytest.mark.parametrize('error', [ diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 3b84fc254..7784f3f77 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -4,6 +4,7 @@ import numpy as np import pandas as pd import pytest +from freqtrade.data.dataprovider import DataProvider from freqtrade.strategy import merge_informative_pair, stoploss_from_open, timeframe_to_minutes @@ -132,3 +133,57 @@ def test_stoploss_from_open(): assert stoploss == 0 else: assert isclose(stop_price, expected_stop_price, rel_tol=0.00001) + + +def test_informative_decorator(mocker, default_conf): + test_data_5m = generate_test_data('5m', 40) + test_data_30m = generate_test_data('30m', 40) + test_data_1h = generate_test_data('1h', 40) + data = { + ('XRP/USDT', '5m'): test_data_5m, + ('XRP/USDT', '30m'): test_data_30m, + ('XRP/USDT', '1h'): test_data_1h, + ('LTC/USDT', '5m'): test_data_5m, + ('LTC/USDT', '30m'): test_data_30m, + ('LTC/USDT', '1h'): test_data_1h, + ('BTC/USDT', '30m'): test_data_30m, + ('BTC/USDT', '5m'): test_data_5m, + ('BTC/USDT', '1h'): test_data_1h, + ('ETH/USDT', '1h'): test_data_1h, + ('ETH/USDT', '30m'): test_data_30m, + ('ETH/BTC', '1h'): test_data_1h, + } + from .strats.informative_decorator_strategy import InformativeDecoratorTest + default_conf['stake_currency'] = 'USDT' + InformativeDecoratorTest.dp = DataProvider({}, None, None) + mocker.patch.object(InformativeDecoratorTest.dp, 'current_whitelist', return_value=[ + 'XRP/USDT', 'LTC/USDT' + ]) + strategy = InformativeDecoratorTest(config=default_conf) + + assert len(strategy._ft_informative) == 8 + informative_pairs = [('XRP/USDT', '1h'), ('LTC/USDT', '1h'), ('XRP/USDT', '30m'), + ('LTC/USDT', '30m'), ('BTC/USDT', '1h'), ('BTC/USDT', '30m'), + ('BTC/USDT', '5m'), ('ETH/BTC', '1h'), ('ETH/USDT', '30m')] + for inf_pair in informative_pairs: + assert inf_pair in strategy.gather_informative_pairs() + + def test_historic_ohlcv(pair, timeframe): + return data[(pair, timeframe or strategy.timeframe)].copy() + mocker.patch('freqtrade.data.dataprovider.DataProvider.historic_ohlcv', + side_effect=test_historic_ohlcv) + + analyzed = strategy.advise_all_indicators( + {p: data[(p, strategy.timeframe)] for p in ('XRP/USDT', 'LTC/USDT')}) + expected_columns = [ + 'rsi_1h', 'rsi_30m', # Stacked informative decorators + 'btc_rsi_1h', # BTC 1h informative + 'rsi_BTC_USDT_btc_usdt_BTC/USDT_30m', # Column formatting + 'rsi_from_callable', # Custom column formatter + 'eth_btc_rsi_1h', # Quote currency not matching stake currency + 'rsi', 'rsi_less', # Non-informative columns + 'rsi_5m', # Manual informative dataframe + ] + for _, dataframe in analyzed.items(): + for col in expected_columns: + assert col in dataframe.columns diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 2cbc9d0c6..3a30a824a 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -35,7 +35,7 @@ def test_search_all_strategies_no_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=False) assert isinstance(strategies, list) - assert len(strategies) == 3 + assert len(strategies) == 4 assert isinstance(strategies[0], dict) @@ -43,10 +43,10 @@ def test_search_all_strategies_with_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) assert isinstance(strategies, list) - assert len(strategies) == 4 + assert len(strategies) == 5 # with enum_failed=True search_all_objects() shall find 2 good strategies # and 1 which fails to load - assert len([x for x in strategies if x['class'] is not None]) == 3 + assert len([x for x in strategies if x['class'] is not None]) == 4 assert len([x for x in strategies if x['class'] is None]) == 1 From f2a1d9d2fc8efe238e47f5404dd6c924511e683b Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sun, 5 Sep 2021 09:54:05 +0300 Subject: [PATCH 098/239] [SQUASH] Address PR comments. --- docs/strategy-customization.md | 56 ++++++++++++++++++++------- freqtrade/strategy/interface.py | 18 ++++----- freqtrade/strategy/strategy_helper.py | 32 +++++++++------ 3 files changed, 72 insertions(+), 34 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 526c111c5..f2bf6cf7c 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -681,9 +681,47 @@ In some situations it may be confusing to deal with stops relative to current ra ### *@informative()* +``` python +def informative(timeframe: str, asset: str = '', + fmt: Optional[Union[str, Callable[[KwArg(str)], str]]] = None, + ffill: bool = True) -> Callable[[PopulateIndicators], PopulateIndicators]: + """ + A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to + define informative indicators. + + Example usage: + + @informative('1h') + def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + return dataframe + + :param timeframe: Informative timeframe. Must always be equal or higher than strategy timeframe. + :param asset: Informative asset, for example BTC, BTC/USDT, ETH/BTC. Do not specify to use + current pair. + :param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not + specified, defaults to: + * {base}_{column}_{timeframe} if asset is specified and quote currency does match stake + curerncy. + * {base}_{quote}_{column}_{timeframe} if asset is specified and quote currency does not match + stake curerncy. + * {column}_{timeframe} if asset is not specified. + Format string supports these format variables: + * {asset} - full name of the asset, for example 'BTC/USDT'. + * {base} - base currency in lower case, for example 'eth'. + * {BASE} - same as {base}, except in upper case. + * {quote} - quote currency in lower case, for example 'usdt'. + * {QUOTE} - same as {quote}, except in upper case. + * {column} - name of dataframe column. + * {timeframe} - timeframe of informative dataframe. + :param ffill: ffill dataframe after merging informative pair. + """ +``` + In most common case it is possible to easily define informative pairs by using a decorator. All decorated `populate_indicators_*` methods run in isolation, not having access to data from other informative pairs, in the end all informative dataframes are merged and passed to main `populate_indicators()` method. -When hyperopting, please follow instructions of [optimizing an indicator parameter](hyperopt.md#optimizing-an-indicator-parameter). +When hyperopting, use of hyperoptable parameter `.value` attribute is not supported. Please use `.range` attribute. See [optimizing an indicator parameter](hyperopt.md#optimizing-an-indicator-parameter) +for more information. ??? Example "Fast and easy way to define informative pairs" @@ -725,17 +763,9 @@ When hyperopting, please follow instructions of [optimizing an indicator paramet return dataframe # Define BTC/STAKE informative pair. A custom formatter may be specified for formatting - # column names. Format string supports these format variables: - # * {asset} - full name of the asset, for example 'BTC/USDT'. - # * {base} - base currency in lower case, for example 'eth'. - # * {BASE} - same as {base}, except in upper case. - # * {quote} - quote currency in lower case, for example 'usdt'. - # * {QUOTE} - same as {quote}, except in upper case. - # * {column} - name of dataframe column. - # * {timeframe} - timeframe of informative dataframe. - # A callable `fmt(**kwargs) -> str` may be specified, to implement custom formatting. - # Available in populate_indicators and other methods as 'rsi_upper'. - @informative('1h', 'BTC/{stake}', '{name}') + # column names. A callable `fmt(**kwargs) -> str` may be specified, to implement custom + # formatting. Available in populate_indicators and other methods as 'rsi_upper'. + @informative('1h', 'BTC/{stake}', '{column}') def populate_indicators_btc_1h_2(self, dataframe: DataFrame, metadata: dict) -> DataFrame: dataframe['rsi_upper'] = ta.RSI(dataframe, timeperiod=14) return dataframe @@ -749,8 +779,6 @@ When hyperopting, please follow instructions of [optimizing an indicator paramet ``` - See docstring of `@informative()` decorator for more information. - !!! Note Do not use `@informative` decorator if you need to use data of one informative pair when generating another informative pair. Instead, define informative pairs manually as described [in the DataProvider section](#complete-data-provider-sample). diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 8e8b8b404..0546deb01 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -6,7 +6,7 @@ import logging import warnings from abc import ABC, abstractmethod from datetime import datetime, timedelta, timezone -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union import arrow from pandas import DataFrame @@ -19,7 +19,8 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.exchange.exchange import timeframe_to_next_date from freqtrade.persistence import PairLocks, Trade from freqtrade.strategy.hyper import HyperStrategyMixin -from freqtrade.strategy.strategy_helper import (InformativeData, _create_and_merge_informative_pair, +from freqtrade.strategy.strategy_helper import (InformativeData, PopulateIndicators, + _create_and_merge_informative_pair, _format_pair_name) from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets @@ -138,20 +139,23 @@ class IStrategy(ABC, HyperStrategyMixin): # Gather informative pairs from @informative-decorated methods. self._ft_informative: Dict[ - Tuple[str, str], Tuple[InformativeData, - Callable[[Any, DataFrame, dict], DataFrame]]] = {} + Tuple[str, str], Tuple[InformativeData, PopulateIndicators]] = {} for attr_name in dir(self.__class__): cls_method = getattr(self.__class__, attr_name) if not callable(cls_method): continue - ft_informative = getattr(cls_method, '_ft_informative', []) + ft_informative = getattr(cls_method, '_ft_informative', None) if not isinstance(ft_informative, list): # Type check is required because mocker would return a mock object that evaluates to # True, confusing this code. continue + strategy_timeframe_minutes = timeframe_to_minutes(self.timeframe) for informative_data in ft_informative: asset = informative_data.asset timeframe = informative_data.timeframe + if timeframe_to_minutes(timeframe) < strategy_timeframe_minutes: + raise OperationalException('Informative timeframe must be equal or higher than ' + 'strategy timeframe!') if asset: pair = _format_pair_name(self.config, asset) if (pair, timeframe) in self._ft_informative: @@ -165,10 +169,6 @@ class IStrategy(ABC, HyperStrategyMixin): f'not be defined more than once!') self._ft_informative[(pair, timeframe)] = (informative_data, cls_method) - def _format_pair(self, pair: str) -> str: - return pair.format(stake_currency=self.config['stake_currency'], - stake=self.config['stake_currency']).upper() - @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index aa828d330..64d9bdea8 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -8,6 +8,9 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes +PopulateIndicators = Callable[[Any, DataFrame, dict], DataFrame] + + class InformativeData(NamedTuple): asset: Optional[str] timeframe: str @@ -118,8 +121,7 @@ def stoploss_from_absolute(stop_rate: float, current_rate: float) -> float: def informative(timeframe: str, asset: str = '', fmt: Optional[Union[str, Callable[[KwArg(str)], str]]] = None, - ffill: bool = True) -> Callable[[Callable[[Any, DataFrame, dict], DataFrame]], - Callable[[Any, DataFrame, dict], DataFrame]]: + ffill: bool = True) -> Callable[[PopulateIndicators], PopulateIndicators]: """ A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to define informative indicators. @@ -131,24 +133,32 @@ def informative(timeframe: str, asset: str = '', dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) return dataframe - :param timeframe: Informative timeframe. Must always be higher than strategy timeframe. + :param timeframe: Informative timeframe. Must always be equal or higher than strategy timeframe. :param asset: Informative asset, for example BTC, BTC/USDT, ETH/BTC. Do not specify to use current pair. :param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not - specified, defaults to {asset}_{name}_{timeframe} if asset is specified, or {name}_{timeframe} - otherwise. - * {asset}: name of informative asset, provided in lower-case, with / replaced with _. Stake - currency is not included in this string. - * {name}: user-specified dataframe column name. - * {timeframe}: informative timeframe. - :param ffill: ffill dataframe after mering informative pair. + specified, defaults to: + * {base}_{column}_{timeframe} if asset is specified and quote currency does match stake + curerncy. + * {base}_{quote}_{column}_{timeframe} if asset is specified and quote currency does not match + stake curerncy. + * {column}_{timeframe} if asset is not specified. + Format string supports these format variables: + * {asset} - full name of the asset, for example 'BTC/USDT'. + * {base} - base currency in lower case, for example 'eth'. + * {BASE} - same as {base}, except in upper case. + * {quote} - quote currency in lower case, for example 'usdt'. + * {QUOTE} - same as {quote}, except in upper case. + * {column} - name of dataframe column. + * {timeframe} - timeframe of informative dataframe. + :param ffill: ffill dataframe after merging informative pair. """ _asset = asset _timeframe = timeframe _fmt = fmt _ffill = ffill - def decorator(fn: Callable[[Any, DataFrame, dict], DataFrame]): + def decorator(fn: PopulateIndicators): informative_pairs = getattr(fn, '_ft_informative', []) informative_pairs.append(InformativeData(_asset, _timeframe, _fmt, _ffill)) setattr(fn, '_ft_informative', informative_pairs) From dfa61b7ad2297a94b0b26309ca74c64a7c1d08a7 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Tue, 7 Sep 2021 15:40:53 +0300 Subject: [PATCH 099/239] [SQUASH] Fix informatives for each pair not being created because dataprovider was not available. Fix not being able to have informative dataframe of a pair in whitelist. --- docs/strategy-customization.md | 4 +-- freqtrade/freqtradebot.py | 10 ++++--- freqtrade/optimize/backtesting.py | 3 +- freqtrade/optimize/edge_cli.py | 3 ++ freqtrade/strategy/interface.py | 38 +++++++++++++++---------- freqtrade/strategy/strategy_helper.py | 13 ++++----- tests/strategy/test_strategy_helpers.py | 9 +++--- 7 files changed, 47 insertions(+), 33 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index f2bf6cf7c..a994d9acd 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -702,9 +702,9 @@ def informative(timeframe: str, asset: str = '', :param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not specified, defaults to: * {base}_{column}_{timeframe} if asset is specified and quote currency does match stake - curerncy. + currency. * {base}_{quote}_{column}_{timeframe} if asset is specified and quote currency does not match - stake curerncy. + stake currency. * {column}_{timeframe} if asset is not specified. Format string supports these format variables: * {asset} - full name of the asset, for example 'BTC/USDT'. diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index bdc438c9a..b79916639 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -83,10 +83,12 @@ class FreqtradeBot(LoggingMixin): self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) - # Attach Dataprovider to Strategy baseclass - IStrategy.dp = self.dataprovider - # Attach Wallets to Strategy baseclass - IStrategy.wallets = self.wallets + # Attach Dataprovider to strategy instance + self.strategy.dp = self.dataprovider + # Attach Wallets to strategy instance + self.strategy.wallets = self.wallets + # Late initialization (may depend on dp/wallets) + self.strategy._initialize() # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3e06bfa1b..ef491ae5e 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -154,11 +154,12 @@ class Backtesting: self.strategy: IStrategy = strategy strategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass - IStrategy.wallets = self.wallets + strategy.wallets = self.wallets # Set stoploss_on_exchange to false for backtesting, # since a "perfect" stoploss-sell is assumed anyway # And the regular "stoploss" function would not apply to that case self.strategy.order_types['stoploss_on_exchange'] = False + strategy._initialize() def _load_protections(self, strategy: IStrategy): if self.config.get('enable_protections', False): diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index 417faa685..abb5ca635 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -8,6 +8,7 @@ from typing import Any, Dict from freqtrade import constants from freqtrade.configuration import TimeRange, validate_config_consistency +from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge from freqtrade.optimize.optimize_reports import generate_edge_table from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -33,6 +34,8 @@ class EdgeCli: self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) self.strategy = StrategyResolver.load_strategy(self.config) + self.strategy.dp = DataProvider(config, None) + self.strategy._initialize() validate_config_consistency(self.config) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 0546deb01..951979212 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -137,9 +137,13 @@ class IStrategy(ABC, HyperStrategyMixin): self._last_candle_seen_per_pair: Dict[str, datetime] = {} super().__init__(config) + def _initialize(self): + """ + Late initialization tasks, which may depend on availability of dataprovider/wallets/etc. + """ # Gather informative pairs from @informative-decorated methods. self._ft_informative: Dict[ - Tuple[str, str], Tuple[InformativeData, PopulateIndicators]] = {} + Tuple[str, str], List[Tuple[InformativeData, PopulateIndicators]]] = {} for attr_name in dir(self.__class__): cls_method = getattr(self.__class__, attr_name) if not callable(cls_method): @@ -158,16 +162,19 @@ class IStrategy(ABC, HyperStrategyMixin): 'strategy timeframe!') if asset: pair = _format_pair_name(self.config, asset) - if (pair, timeframe) in self._ft_informative: - raise OperationalException(f'Informative pair {pair} {timeframe} can not ' - f'be defined more than once!') - self._ft_informative[(pair, timeframe)] = (informative_data, cls_method) - elif self.dp is not None: + try: + self._ft_informative[(pair, timeframe)].append( + (informative_data, cls_method)) + except KeyError: + self._ft_informative[(pair, timeframe)] = [(informative_data, cls_method)] + else: for pair in self.dp.current_whitelist(): - if (pair, timeframe) in self._ft_informative: - raise OperationalException(f'Informative pair {pair} {timeframe} can ' - f'not be defined more than once!') - self._ft_informative[(pair, timeframe)] = (informative_data, cls_method) + try: + self._ft_informative[(pair, timeframe)].append( + (informative_data, cls_method)) + except KeyError: + self._ft_informative[(pair, timeframe)] = \ + [(informative_data, cls_method)] @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -838,11 +845,12 @@ class IStrategy(ABC, HyperStrategyMixin): logger.debug(f"Populating indicators for pair {metadata.get('pair')}.") # call populate_indicators_Nm() which were tagged with @informative decorator. - for (pair, timeframe), (informative_data, populate_fn) in self._ft_informative.items(): - if not informative_data.asset and pair != metadata['pair']: - continue - dataframe = _create_and_merge_informative_pair( - self, dataframe, metadata, informative_data, populate_fn) + for (pair, timeframe), informatives in self._ft_informative.items(): + for (informative_data, populate_fn) in informatives: + if not informative_data.asset and pair != metadata['pair']: + continue + dataframe = _create_and_merge_informative_pair( + self, dataframe, metadata, informative_data, populate_fn) if self._populate_fun_len == 2: warnings.warn("deprecated - check out the Sample strategy to see " diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index 64d9bdea8..a4023f953 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -139,9 +139,9 @@ def informative(timeframe: str, asset: str = '', :param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not specified, defaults to: * {base}_{column}_{timeframe} if asset is specified and quote currency does match stake - curerncy. + currency. * {base}_{quote}_{column}_{timeframe} if asset is specified and quote currency does not match - stake curerncy. + stake currency. * {column}_{timeframe} if asset is not specified. Format string supports these format variables: * {asset} - full name of the asset, for example 'BTC/USDT'. @@ -203,11 +203,10 @@ def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, # fmt='{base}_{quote}_{column}_{timeframe}' format or similar. if not fmt: fmt = '{column}_{timeframe}' # Informatives of current pair - if asset != metadata['pair']: - if quote == config['stake_currency']: - fmt = '{base}_' + fmt # Informatives of other pair - else: - fmt = '{base}_{quote}_' + fmt # Informatives of different quote currency + if quote != config['stake_currency']: + fmt = '{quote}_' + fmt # Informatives of different quote currency + if informative_data.asset: + fmt = '{base}_' + fmt # Informatives of other pair inf_metadata = {'pair': asset, 'timeframe': timeframe} inf_dataframe = dp.get_pair_dataframe(asset, timeframe) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 7784f3f77..0ee554ede 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -155,11 +155,12 @@ def test_informative_decorator(mocker, default_conf): } from .strats.informative_decorator_strategy import InformativeDecoratorTest default_conf['stake_currency'] = 'USDT' - InformativeDecoratorTest.dp = DataProvider({}, None, None) - mocker.patch.object(InformativeDecoratorTest.dp, 'current_whitelist', return_value=[ - 'XRP/USDT', 'LTC/USDT' - ]) strategy = InformativeDecoratorTest(config=default_conf) + strategy.dp = DataProvider({}, None, None) + mocker.patch.object(strategy.dp, 'current_whitelist', return_value=[ + 'XRP/USDT', 'LTC/USDT', 'BTC/USDT' + ]) + strategy._initialize() assert len(strategy._ft_informative) == 8 informative_pairs = [('XRP/USDT', '1h'), ('LTC/USDT', '1h'), ('XRP/USDT', '30m'), From f81df19b934b738bf148f51fdb5a9c7ab1dfb34f Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Tue, 7 Sep 2021 15:53:12 +0300 Subject: [PATCH 100/239] [TMP] Make tests not fail for now. --- freqtrade/strategy/interface.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 951979212..6e312e15d 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -424,7 +424,8 @@ class IStrategy(ABC, HyperStrategyMixin): Internal method which gathers all informative pairs (user or automatically defined). """ informative_pairs = self.informative_pairs() - informative_pairs += list(self._ft_informative.keys()) + if hasattr(self, '_ft_informative'): + informative_pairs += list(self._ft_informative.keys()) return list(set(informative_pairs)) def get_strategy_name(self) -> str: @@ -844,13 +845,14 @@ class IStrategy(ABC, HyperStrategyMixin): """ logger.debug(f"Populating indicators for pair {metadata.get('pair')}.") - # call populate_indicators_Nm() which were tagged with @informative decorator. - for (pair, timeframe), informatives in self._ft_informative.items(): - for (informative_data, populate_fn) in informatives: - if not informative_data.asset and pair != metadata['pair']: - continue - dataframe = _create_and_merge_informative_pair( - self, dataframe, metadata, informative_data, populate_fn) + if hasattr(self, '_ft_informative'): + # call populate_indicators_Nm() which were tagged with @informative decorator. + for (pair, timeframe), informatives in self._ft_informative.items(): + for (informative_data, populate_fn) in informatives: + if not informative_data.asset and pair != metadata['pair']: + continue + dataframe = _create_and_merge_informative_pair( + self, dataframe, metadata, informative_data, populate_fn) if self._populate_fun_len == 2: warnings.warn("deprecated - check out the Sample strategy to see " From 5dc78a0c66f385edd14db16a806e1f75bd453e83 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Fri, 10 Sep 2021 09:36:52 +0300 Subject: [PATCH 101/239] [SQUASH] Get rid of _initialize() and fix informatives for dynamic pairlists. --- freqtrade/freqtradebot.py | 2 - freqtrade/optimize/backtesting.py | 1 - freqtrade/optimize/edge_cli.py | 1 - freqtrade/strategy/interface.py | 56 ++++++++----------------- freqtrade/strategy/strategy_helper.py | 17 ++++---- tests/strategy/test_strategy_helpers.py | 3 +- 6 files changed, 27 insertions(+), 53 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index b79916639..1cb8988ff 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -87,8 +87,6 @@ class FreqtradeBot(LoggingMixin): self.strategy.dp = self.dataprovider # Attach Wallets to strategy instance self.strategy.wallets = self.wallets - # Late initialization (may depend on dp/wallets) - self.strategy._initialize() # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ef491ae5e..79c861ee8 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -159,7 +159,6 @@ class Backtesting: # since a "perfect" stoploss-sell is assumed anyway # And the regular "stoploss" function would not apply to that case self.strategy.order_types['stoploss_on_exchange'] = False - strategy._initialize() def _load_protections(self, strategy: IStrategy): if self.config.get('enable_protections', False): diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index abb5ca635..f211da750 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -35,7 +35,6 @@ class EdgeCli: self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) self.strategy = StrategyResolver.load_strategy(self.config) self.strategy.dp = DataProvider(config, None) - self.strategy._initialize() validate_config_consistency(self.config) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 6e312e15d..00c56f5df 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -121,7 +121,7 @@ class IStrategy(ABC, HyperStrategyMixin): # Class level variables (intentional) containing # the dataprovider (dp) (access to other candles, historic data, ...) # and wallets - access to the current balance. - dp: Optional[DataProvider] = None + dp: DataProvider wallets: Optional[Wallets] = None # Filled from configuration stake_currency: str @@ -137,44 +137,23 @@ class IStrategy(ABC, HyperStrategyMixin): self._last_candle_seen_per_pair: Dict[str, datetime] = {} super().__init__(config) - def _initialize(self): - """ - Late initialization tasks, which may depend on availability of dataprovider/wallets/etc. - """ # Gather informative pairs from @informative-decorated methods. - self._ft_informative: Dict[ - Tuple[str, str], List[Tuple[InformativeData, PopulateIndicators]]] = {} + self._ft_informative: List[Tuple[InformativeData, PopulateIndicators]] = [] for attr_name in dir(self.__class__): cls_method = getattr(self.__class__, attr_name) if not callable(cls_method): continue - ft_informative = getattr(cls_method, '_ft_informative', None) - if not isinstance(ft_informative, list): + informative_data_list = getattr(cls_method, '_ft_informative', None) + if not isinstance(informative_data_list, list): # Type check is required because mocker would return a mock object that evaluates to # True, confusing this code. continue strategy_timeframe_minutes = timeframe_to_minutes(self.timeframe) - for informative_data in ft_informative: - asset = informative_data.asset - timeframe = informative_data.timeframe - if timeframe_to_minutes(timeframe) < strategy_timeframe_minutes: + for informative_data in informative_data_list: + if timeframe_to_minutes(informative_data.timeframe) < strategy_timeframe_minutes: raise OperationalException('Informative timeframe must be equal or higher than ' 'strategy timeframe!') - if asset: - pair = _format_pair_name(self.config, asset) - try: - self._ft_informative[(pair, timeframe)].append( - (informative_data, cls_method)) - except KeyError: - self._ft_informative[(pair, timeframe)] = [(informative_data, cls_method)] - else: - for pair in self.dp.current_whitelist(): - try: - self._ft_informative[(pair, timeframe)].append( - (informative_data, cls_method)) - except KeyError: - self._ft_informative[(pair, timeframe)] = \ - [(informative_data, cls_method)] + self._ft_informative.append((informative_data, cls_method)) @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -424,8 +403,13 @@ class IStrategy(ABC, HyperStrategyMixin): Internal method which gathers all informative pairs (user or automatically defined). """ informative_pairs = self.informative_pairs() - if hasattr(self, '_ft_informative'): - informative_pairs += list(self._ft_informative.keys()) + for inf_data, _ in self._ft_informative: + if inf_data.asset: + pair_tf = (_format_pair_name(self.config, inf_data.asset), inf_data.timeframe) + informative_pairs.append(pair_tf) + else: + for pair in self.dp.current_whitelist(): + informative_pairs.append((pair, inf_data.timeframe)) return list(set(informative_pairs)) def get_strategy_name(self) -> str: @@ -845,14 +829,10 @@ class IStrategy(ABC, HyperStrategyMixin): """ logger.debug(f"Populating indicators for pair {metadata.get('pair')}.") - if hasattr(self, '_ft_informative'): - # call populate_indicators_Nm() which were tagged with @informative decorator. - for (pair, timeframe), informatives in self._ft_informative.items(): - for (informative_data, populate_fn) in informatives: - if not informative_data.asset and pair != metadata['pair']: - continue - dataframe = _create_and_merge_informative_pair( - self, dataframe, metadata, informative_data, populate_fn) + # call populate_indicators_Nm() which were tagged with @informative decorator. + for inf_data, populate_fn in self._ft_informative: + dataframe = _create_and_merge_informative_pair( + self, dataframe, metadata, inf_data, populate_fn) if self._populate_fun_len == 2: warnings.warn("deprecated - check out the Sample strategy to see " diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index a4023f953..15c6d8a69 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -171,14 +171,13 @@ def _format_pair_name(config, pair: str) -> str: stake=config['stake_currency']).upper() -def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, - metadata: dict, informative_data: InformativeData, - populate_indicators: Callable[[Any, DataFrame, dict], - DataFrame]): - asset = informative_data.asset or '' - timeframe = informative_data.timeframe - fmt = informative_data.fmt - ffill = informative_data.ffill +def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, metadata: dict, + inf_data: InformativeData, + populate_indicators: PopulateIndicators): + asset = inf_data.asset or '' + timeframe = inf_data.timeframe + fmt = inf_data.fmt + ffill = inf_data.ffill config = strategy.config dp = strategy.dp @@ -205,7 +204,7 @@ def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, fmt = '{column}_{timeframe}' # Informatives of current pair if quote != config['stake_currency']: fmt = '{quote}_' + fmt # Informatives of different quote currency - if informative_data.asset: + if inf_data.asset: fmt = '{base}_' + fmt # Informatives of other pair inf_metadata = {'pair': asset, 'timeframe': timeframe} diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 0ee554ede..95ca0416f 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -160,9 +160,8 @@ def test_informative_decorator(mocker, default_conf): mocker.patch.object(strategy.dp, 'current_whitelist', return_value=[ 'XRP/USDT', 'LTC/USDT', 'BTC/USDT' ]) - strategy._initialize() - assert len(strategy._ft_informative) == 8 + assert len(strategy._ft_informative) == 6 # Equal to number of decorators used informative_pairs = [('XRP/USDT', '1h'), ('LTC/USDT', '1h'), ('XRP/USDT', '30m'), ('LTC/USDT', '30m'), ('BTC/USDT', '1h'), ('BTC/USDT', '30m'), ('BTC/USDT', '5m'), ('ETH/BTC', '1h'), ('ETH/USDT', '30m')] From bb6ae682fc7d82175196741e31723c5f797ffd2f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Sep 2021 09:59:10 +0200 Subject: [PATCH 102/239] Small simplifications --- docs/strategy-customization.md | 2 +- freqtrade/strategy/strategy_helper.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index a994d9acd..800dd9326 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -783,7 +783,7 @@ for more information. Do not use `@informative` decorator if you need to use data of one informative pair when generating another informative pair. Instead, define informative pairs manually as described [in the DataProvider section](#complete-data-provider-sample). -!!! Warning +!!! Warning "Duplicate method names" Methods tagged with `@informative()` decorator must always have unique names! Re-using same name (for example when copy-pasting already defined informative method) will overwrite previously defined method and not produce any errors due to limitations of Python programming language. In such cases you will find that indicators created in earlier-defined methods are not available in the dataframe. Carefully review method names and make sure they are unique! diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index 15c6d8a69..746d656df 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -177,9 +177,7 @@ def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, metadata: asset = inf_data.asset or '' timeframe = inf_data.timeframe fmt = inf_data.fmt - ffill = inf_data.ffill config = strategy.config - dp = strategy.dp if asset: # Insert stake currency if needed. @@ -208,7 +206,7 @@ def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, metadata: fmt = '{base}_' + fmt # Informatives of other pair inf_metadata = {'pair': asset, 'timeframe': timeframe} - inf_dataframe = dp.get_pair_dataframe(asset, timeframe) + inf_dataframe = strategy.dp.get_pair_dataframe(asset, timeframe) inf_dataframe = populate_indicators(strategy, inf_dataframe, inf_metadata) formatter: Any = None @@ -233,6 +231,6 @@ def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, metadata: raise OperationalException(f'Duplicate column name {date_column} exists in ' f'dataframe! Ensure column names are unique!') dataframe = merge_informative_pair(dataframe, inf_dataframe, strategy.timeframe, timeframe, - ffill=ffill, append_timeframe=False, + ffill=inf_data.ffill, append_timeframe=False, date_column=date_column) return dataframe From e88c4701bb2c321ed7cd5d6fed7fa6db6f4f41f6 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sun, 12 Sep 2021 18:26:41 +0300 Subject: [PATCH 103/239] [SQUASH] Address PR comments. --- docs/strategy-customization.md | 19 ++- freqtrade/strategy/__init__.py | 5 +- freqtrade/strategy/informative_decorator.py | 134 ++++++++++++++++++++ freqtrade/strategy/interface.py | 11 +- freqtrade/strategy/strategy_helper.py | 132 ------------------- 5 files changed, 152 insertions(+), 149 deletions(-) create mode 100644 freqtrade/strategy/informative_decorator.py diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 800dd9326..671768bfa 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -652,9 +652,7 @@ In some situations it may be confusing to deal with stops relative to current ra ??? Example "Returning a stoploss using absolute price from the custom stoploss function" - Say the open price was $100, and `current_price` is $121 (`current_profit` will be `0.21`). - - If we want a stop price at $107 price we can call `stoploss_from_absolute(107, current_rate)` which will return `0.1157024793`. 11.57% below $121 is $107, which is the same as 7% above $100. + If we want to trail a stop price at 2xATR below current proce we can call `stoploss_from_absolute(current_rate - (candle['atr'] * 2), current_rate)`. ``` python @@ -664,18 +662,17 @@ In some situations it may be confusing to deal with stops relative to current ra class AwesomeStrategy(IStrategy): - # ... populate_* methods - use_custom_stoploss = True + def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['atr'] = ta.ATR(dataframe, timeperiod=14) + return dataframe + def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, current_rate: float, current_profit: float, **kwargs) -> float: - - # once the profit has risen above 10%, keep the stoploss at 7% above the open price - if current_profit > 0.10: - return stoploss_from_absolute(trade.open_rate * 1.07, current_rate) - - return 1 + dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) + candle = dataframe.iloc[-1].squeeze() + return stoploss_from_absolute(current_rate - (candle['atr'] * 2), current_rate) ``` diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index a7de34916..2ea0ad2b4 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -3,6 +3,7 @@ from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_msecs, timefr timeframe_to_prev_date, timeframe_to_seconds) from freqtrade.strategy.hyper import (BooleanParameter, CategoricalParameter, DecimalParameter, IntParameter, RealParameter) +from freqtrade.strategy.informative_decorator import informative from freqtrade.strategy.interface import IStrategy -from freqtrade.strategy.strategy_helper import (informative, merge_informative_pair, - stoploss_from_absolute, stoploss_from_open) +from freqtrade.strategy.strategy_helper import (merge_informative_pair, stoploss_from_absolute, + stoploss_from_open) diff --git a/freqtrade/strategy/informative_decorator.py b/freqtrade/strategy/informative_decorator.py new file mode 100644 index 000000000..f09e634b0 --- /dev/null +++ b/freqtrade/strategy/informative_decorator.py @@ -0,0 +1,134 @@ +from typing import Any, Callable, NamedTuple, Optional, Union + +from mypy_extensions import KwArg +from pandas import DataFrame + +from freqtrade.exceptions import OperationalException +from freqtrade.strategy.strategy_helper import merge_informative_pair + + +PopulateIndicators = Callable[[Any, DataFrame, dict], DataFrame] + + +class InformativeData(NamedTuple): + asset: Optional[str] + timeframe: str + fmt: Union[str, Callable[[KwArg(str)], str], None] + ffill: bool + + +def informative(timeframe: str, asset: str = '', + fmt: Optional[Union[str, Callable[[KwArg(str)], str]]] = None, + ffill: bool = True) -> Callable[[PopulateIndicators], PopulateIndicators]: + """ + A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to + define informative indicators. + + Example usage: + + @informative('1h') + def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + return dataframe + + :param timeframe: Informative timeframe. Must always be equal or higher than strategy timeframe. + :param asset: Informative asset, for example BTC, BTC/USDT, ETH/BTC. Do not specify to use + current pair. + :param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not + specified, defaults to: + * {base}_{column}_{timeframe} if asset is specified and quote currency does match stake + currency. + * {base}_{quote}_{column}_{timeframe} if asset is specified and quote currency does not match + stake currency. + * {column}_{timeframe} if asset is not specified. + Format string supports these format variables: + * {asset} - full name of the asset, for example 'BTC/USDT'. + * {base} - base currency in lower case, for example 'eth'. + * {BASE} - same as {base}, except in upper case. + * {quote} - quote currency in lower case, for example 'usdt'. + * {QUOTE} - same as {quote}, except in upper case. + * {column} - name of dataframe column. + * {timeframe} - timeframe of informative dataframe. + :param ffill: ffill dataframe after merging informative pair. + """ + _asset = asset + _timeframe = timeframe + _fmt = fmt + _ffill = ffill + + def decorator(fn: PopulateIndicators): + informative_pairs = getattr(fn, '_ft_informative', []) + informative_pairs.append(InformativeData(_asset, _timeframe, _fmt, _ffill)) + setattr(fn, '_ft_informative', informative_pairs) + return fn + return decorator + + +def _format_pair_name(config, pair: str) -> str: + return pair.format(stake_currency=config['stake_currency'], + stake=config['stake_currency']).upper() + + +def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, metadata: dict, + inf_data: InformativeData, + populate_indicators: PopulateIndicators): + asset = inf_data.asset or '' + timeframe = inf_data.timeframe + fmt = inf_data.fmt + config = strategy.config + + if asset: + # Insert stake currency if needed. + asset = _format_pair_name(config, asset) + else: + # Not specifying an asset will define informative dataframe for current pair. + asset = metadata['pair'] + + if '/' in asset: + base, quote = asset.split('/') + else: + # When futures are supported this may need reevaluation. + # base, quote = asset, None + raise OperationalException('Not implemented.') + + # Default format. This optimizes for the common case: informative pairs using same stake + # currency. When quote currency matches stake currency, column name will omit base currency. + # This allows easily reconfiguring strategy to use different base currency. In a rare case + # where it is desired to keep quote currency in column name at all times user should specify + # fmt='{base}_{quote}_{column}_{timeframe}' format or similar. + if not fmt: + fmt = '{column}_{timeframe}' # Informatives of current pair + if quote != config['stake_currency']: + fmt = '{quote}_' + fmt # Informatives of different quote currency + if inf_data.asset: + fmt = '{base}_' + fmt # Informatives of other pair + + inf_metadata = {'pair': asset, 'timeframe': timeframe} + inf_dataframe = strategy.dp.get_pair_dataframe(asset, timeframe) + inf_dataframe = populate_indicators(strategy, inf_dataframe, inf_metadata) + + formatter: Any = None + if callable(fmt): + formatter = fmt # A custom user-specified formatter function. + else: + formatter = fmt.format # A default string formatter. + + fmt_args = { + 'BASE': base.upper(), + 'QUOTE': quote.upper(), + 'base': base.lower(), + 'quote': quote.lower(), + 'asset': asset, + 'timeframe': timeframe, + } + inf_dataframe.rename(columns=lambda column: formatter(column=column, **fmt_args), + inplace=True) + + date_column = formatter(column='date', **fmt_args) + if date_column in dataframe.columns: + raise OperationalException(f'Duplicate column name {date_column} exists in ' + f'dataframe! Ensure column names are unique!') + dataframe = merge_informative_pair(dataframe, inf_dataframe, strategy.timeframe, timeframe, + ffill=inf_data.ffill, append_timeframe=False, + date_column=date_column) + return dataframe diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 00c56f5df..7420bd9fd 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -19,9 +19,9 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.exchange.exchange import timeframe_to_next_date from freqtrade.persistence import PairLocks, Trade from freqtrade.strategy.hyper import HyperStrategyMixin -from freqtrade.strategy.strategy_helper import (InformativeData, PopulateIndicators, - _create_and_merge_informative_pair, - _format_pair_name) +from freqtrade.strategy.informative_decorator import (InformativeData, PopulateIndicators, + _create_and_merge_informative_pair, + _format_pair_name) from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets @@ -121,7 +121,7 @@ class IStrategy(ABC, HyperStrategyMixin): # Class level variables (intentional) containing # the dataprovider (dp) (access to other candles, historic data, ...) # and wallets - access to the current balance. - dp: DataProvider + dp: Optional[DataProvider] wallets: Optional[Wallets] = None # Filled from configuration stake_currency: str @@ -408,6 +408,9 @@ class IStrategy(ABC, HyperStrategyMixin): pair_tf = (_format_pair_name(self.config, inf_data.asset), inf_data.timeframe) informative_pairs.append(pair_tf) else: + if not self.dp: + raise OperationalException('@informative decorator with unspecified asset ' + 'requires DataProvider instance.') for pair in self.dp.current_whitelist(): informative_pairs.append((pair, inf_data.timeframe)) return list(set(informative_pairs)) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index 746d656df..f813b39c5 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -1,23 +1,8 @@ -from typing import Any, Callable, NamedTuple, Optional, Union - import pandas as pd -from mypy_extensions import KwArg -from pandas import DataFrame -from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes -PopulateIndicators = Callable[[Any, DataFrame, dict], DataFrame] - - -class InformativeData(NamedTuple): - asset: Optional[str] - timeframe: str - fmt: Union[str, Callable[[KwArg(str)], str], None] - ffill: bool - - def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, timeframe: str, timeframe_inf: str, ffill: bool = True, append_timeframe: bool = True, @@ -117,120 +102,3 @@ def stoploss_from_absolute(stop_rate: float, current_rate: float) -> float: :return: Positive stop loss value relative to current price """ return 1 - (stop_rate / current_rate) - - -def informative(timeframe: str, asset: str = '', - fmt: Optional[Union[str, Callable[[KwArg(str)], str]]] = None, - ffill: bool = True) -> Callable[[PopulateIndicators], PopulateIndicators]: - """ - A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to - define informative indicators. - - Example usage: - - @informative('1h') - def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) - return dataframe - - :param timeframe: Informative timeframe. Must always be equal or higher than strategy timeframe. - :param asset: Informative asset, for example BTC, BTC/USDT, ETH/BTC. Do not specify to use - current pair. - :param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not - specified, defaults to: - * {base}_{column}_{timeframe} if asset is specified and quote currency does match stake - currency. - * {base}_{quote}_{column}_{timeframe} if asset is specified and quote currency does not match - stake currency. - * {column}_{timeframe} if asset is not specified. - Format string supports these format variables: - * {asset} - full name of the asset, for example 'BTC/USDT'. - * {base} - base currency in lower case, for example 'eth'. - * {BASE} - same as {base}, except in upper case. - * {quote} - quote currency in lower case, for example 'usdt'. - * {QUOTE} - same as {quote}, except in upper case. - * {column} - name of dataframe column. - * {timeframe} - timeframe of informative dataframe. - :param ffill: ffill dataframe after merging informative pair. - """ - _asset = asset - _timeframe = timeframe - _fmt = fmt - _ffill = ffill - - def decorator(fn: PopulateIndicators): - informative_pairs = getattr(fn, '_ft_informative', []) - informative_pairs.append(InformativeData(_asset, _timeframe, _fmt, _ffill)) - setattr(fn, '_ft_informative', informative_pairs) - return fn - return decorator - - -def _format_pair_name(config, pair: str) -> str: - return pair.format(stake_currency=config['stake_currency'], - stake=config['stake_currency']).upper() - - -def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, metadata: dict, - inf_data: InformativeData, - populate_indicators: PopulateIndicators): - asset = inf_data.asset or '' - timeframe = inf_data.timeframe - fmt = inf_data.fmt - config = strategy.config - - if asset: - # Insert stake currency if needed. - asset = _format_pair_name(config, asset) - else: - # Not specifying an asset will define informative dataframe for current pair. - asset = metadata['pair'] - - if '/' in asset: - base, quote = asset.split('/') - else: - # When futures are supported this may need reevaluation. - # base, quote = asset, None - raise OperationalException('Not implemented.') - - # Default format. This optimizes for the common case: informative pairs using same stake - # currency. When quote currency matches stake currency, column name will omit base currency. - # This allows easily reconfiguring strategy to use different base currency. In a rare case - # where it is desired to keep quote currency in column name at all times user should specify - # fmt='{base}_{quote}_{column}_{timeframe}' format or similar. - if not fmt: - fmt = '{column}_{timeframe}' # Informatives of current pair - if quote != config['stake_currency']: - fmt = '{quote}_' + fmt # Informatives of different quote currency - if inf_data.asset: - fmt = '{base}_' + fmt # Informatives of other pair - - inf_metadata = {'pair': asset, 'timeframe': timeframe} - inf_dataframe = strategy.dp.get_pair_dataframe(asset, timeframe) - inf_dataframe = populate_indicators(strategy, inf_dataframe, inf_metadata) - - formatter: Any = None - if callable(fmt): - formatter = fmt # A custom user-specified formatter function. - else: - formatter = fmt.format # A default string formatter. - - fmt_args = { - 'BASE': base.upper(), - 'QUOTE': quote.upper(), - 'base': base.lower(), - 'quote': quote.lower(), - 'asset': asset, - 'timeframe': timeframe, - } - inf_dataframe.rename(columns=lambda column: formatter(column=column, **fmt_args), - inplace=True) - - date_column = formatter(column='date', **fmt_args) - if date_column in dataframe.columns: - raise OperationalException(f'Duplicate column name {date_column} exists in ' - f'dataframe! Ensure column names are unique!') - dataframe = merge_informative_pair(dataframe, inf_dataframe, strategy.timeframe, timeframe, - ffill=inf_data.ffill, append_timeframe=False, - date_column=date_column) - return dataframe From 7e6aa9390ae9c970f8b22afb591161492f833807 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sat, 18 Sep 2021 10:17:50 +0300 Subject: [PATCH 104/239] [SQUASH] Unconditionally include quote currency when asset is explicitly specified. Added docs suggesting to use string formatting to make strategy independent of configured stake currency. --- docs/strategy-customization.md | 26 +++++++++++++++++---- freqtrade/strategy/informative_decorator.py | 11 +++------ tests/strategy/test_strategy_helpers.py | 2 +- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 671768bfa..2b22dd274 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -698,10 +698,7 @@ def informative(timeframe: str, asset: str = '', current pair. :param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not specified, defaults to: - * {base}_{column}_{timeframe} if asset is specified and quote currency does match stake - currency. - * {base}_{quote}_{column}_{timeframe} if asset is specified and quote currency does not match - stake currency. + * {base}_{quote}_{column}_{timeframe} if asset is specified. * {column}_{timeframe} if asset is not specified. Format string supports these format variables: * {asset} - full name of the asset, for example 'BTC/USDT'. @@ -746,7 +743,7 @@ for more information. # Define BTC/STAKE informative pair. Available in populate_indicators and other methods as # 'btc_rsi_1h'. Current stake currency should be specified as {stake} format variable # instead of hardcoding actual stake currency. Available in populate_indicators and other - # methods as 'btc_rsi_1h'. + # methods as 'btc_usdt_rsi_1h' (when stake currency is USDT). @informative('1h', 'BTC/{stake}') def populate_indicators_btc_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) @@ -780,6 +777,25 @@ for more information. Do not use `@informative` decorator if you need to use data of one informative pair when generating another informative pair. Instead, define informative pairs manually as described [in the DataProvider section](#complete-data-provider-sample). +!!! Note + Use string formatting when accessing informative dataframes of other pairs. This will allow easily changing stake currency in config without having to adjust strategy code. + + ``` python + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + stake = self.config['stake_currency'] + dataframe.loc[ + ( + (dataframe[f'btc_{stake}_rsi_1h'] < 35) + & + (dataframe['volume'] > 0) + ), + ['buy', 'buy_tag']] = (1, 'buy_signal_rsi') + + return dataframe + ``` + + Alternatively column renaming may be used to remove stake currency from column names: `@informative('1h', 'BTC/{stake}', fmt='{base}_{column}_{timeframe}')`. + !!! Warning "Duplicate method names" Methods tagged with `@informative()` decorator must always have unique names! Re-using same name (for example when copy-pasting already defined informative method) will overwrite previously defined method and not produce any errors due to limitations of Python programming language. In such cases you will find that indicators diff --git a/freqtrade/strategy/informative_decorator.py b/freqtrade/strategy/informative_decorator.py index f09e634b0..c8ebf5989 100644 --- a/freqtrade/strategy/informative_decorator.py +++ b/freqtrade/strategy/informative_decorator.py @@ -36,10 +36,7 @@ def informative(timeframe: str, asset: str = '', current pair. :param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not specified, defaults to: - * {base}_{column}_{timeframe} if asset is specified and quote currency does match stake - currency. - * {base}_{quote}_{column}_{timeframe} if asset is specified and quote currency does not match - stake currency. + * {base}_{quote}_{column}_{timeframe} if asset is specified. * {column}_{timeframe} if asset is not specified. Format string supports these format variables: * {asset} - full name of the asset, for example 'BTC/USDT'. @@ -88,7 +85,7 @@ def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, metadata: base, quote = asset.split('/') else: # When futures are supported this may need reevaluation. - # base, quote = asset, None + # base, quote = asset, '' raise OperationalException('Not implemented.') # Default format. This optimizes for the common case: informative pairs using same stake @@ -98,10 +95,8 @@ def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, metadata: # fmt='{base}_{quote}_{column}_{timeframe}' format or similar. if not fmt: fmt = '{column}_{timeframe}' # Informatives of current pair - if quote != config['stake_currency']: - fmt = '{quote}_' + fmt # Informatives of different quote currency if inf_data.asset: - fmt = '{base}_' + fmt # Informatives of other pair + fmt = '{base}_{quote}_' + fmt # Informatives of other pairs inf_metadata = {'pair': asset, 'timeframe': timeframe} inf_dataframe = strategy.dp.get_pair_dataframe(asset, timeframe) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 95ca0416f..d4206ba8c 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -177,7 +177,7 @@ def test_informative_decorator(mocker, default_conf): {p: data[(p, strategy.timeframe)] for p in ('XRP/USDT', 'LTC/USDT')}) expected_columns = [ 'rsi_1h', 'rsi_30m', # Stacked informative decorators - 'btc_rsi_1h', # BTC 1h informative + 'btc_usdt_rsi_1h', # BTC 1h informative 'rsi_BTC_USDT_btc_usdt_BTC/USDT_30m', # Column formatting 'rsi_from_callable', # Custom column formatter 'eth_btc_rsi_1h', # Quote currency not matching stake currency From e4ca42faec580a3c336298d42402fec0d8e56f93 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sat, 18 Sep 2021 10:18:33 +0300 Subject: [PATCH 105/239] [SQUASH] Update stoploss_from_absolute to behave more like stoploss_from_open and add a test for it. --- freqtrade/strategy/strategy_helper.py | 16 +++++++++++++++- tests/strategy/test_strategy_helpers.py | 11 ++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index f813b39c5..175bcaccb 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -97,8 +97,22 @@ def stoploss_from_absolute(stop_rate: float, current_rate: float) -> float: """ Given current price and desired stop price, return a stop loss value that is relative to current price. + + The requested stop can be positive for a stop above the open price, or negative for + a stop below the open price. The return value is always >= 0. + + Returns 0 if the resulting stop price would be above the current price. + :param stop_rate: Stop loss price. :param current_rate: Current asset price. :return: Positive stop loss value relative to current price """ - return 1 - (stop_rate / current_rate) + + # formula is undefined for current_rate 0, return maximum value + if current_rate == 0: + return 1 + + stoploss = 1 - (stop_rate / current_rate) + + # negative stoploss values indicate the requested stop price is higher than the current price + return max(stoploss, 0.0) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index d4206ba8c..9132382fa 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -5,7 +5,8 @@ import pandas as pd import pytest from freqtrade.data.dataprovider import DataProvider -from freqtrade.strategy import merge_informative_pair, stoploss_from_open, timeframe_to_minutes +from freqtrade.strategy import (merge_informative_pair, stoploss_from_open, timeframe_to_minutes, + stoploss_from_absolute) def generate_test_data(timeframe: str, size: int): @@ -135,6 +136,14 @@ def test_stoploss_from_open(): assert isclose(stop_price, expected_stop_price, rel_tol=0.00001) +def test_stoploss_from_absolute(): + assert stoploss_from_absolute(90, 100) == 1 - (90 / 100) + assert stoploss_from_absolute(100, 100) == 0 + assert stoploss_from_absolute(110, 100) == 0 + assert stoploss_from_absolute(100, 0) == 1 + assert stoploss_from_absolute(0, 100) == 1 + + def test_informative_decorator(mocker, default_conf): test_data_5m = generate_test_data('5m', 40) test_data_30m = generate_test_data('30m', 40) From 216f75bbb91b8d72790f386dfc8b05340ade3c3d Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 18 Sep 2021 02:53:34 -0600 Subject: [PATCH 106/239] parametrized test_sell_profit_only , test__safe_exit_amount, test_order_book_bid_strategy1 --- tests/test_freqtradebot.py | 244 ++++++++++--------------------------- 1 file changed, 65 insertions(+), 179 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index eb3c77cc7..5268d40ec 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3128,16 +3128,28 @@ def test_execute_trade_exit_insufficient_funds_error(default_conf, ticker, fee, assert mock_insuf.call_count == 1 -def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, limit_buy_order_open, - fee, mocker) -> None: +@pytest.mark.parametrize('profit_only,bid,ask,handle_first,handle_second,sell_type', [ + # Enable profit + (True, 0.00001172, 0.00001173, False, True, SellType.SELL_SIGNAL.value), + # Disable profit + (False, 0.00002172, 0.00002173, True, False, SellType.SELL_SIGNAL.value), + # Enable loss + # * Shouldn't this be SellType.STOP_LOSS.value + (True, 0.00000172, 0.00000173, False, False, None), + # Disable loss + (False, 0.00000172, 0.00000173, True, False, SellType.SELL_SIGNAL.value), +]) +def test_sell_profit_only_enable_profit( + default_conf, limit_buy_order, limit_buy_order_open, + fee, mocker, profit_only, bid, ask, handle_first, handle_second, sell_type) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 + 'bid': bid, + 'ask': ask, + 'last': bid }), create_order=MagicMock(side_effect=[ limit_buy_order_open, @@ -3147,128 +3159,29 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, limit_buy ) default_conf.update({ 'use_sell_signal': True, - 'sell_profit_only': True, + 'sell_profit_only': profit_only, 'sell_profit_offset': 0.1, }) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - + if sell_type == SellType.SELL_SIGNAL.value: + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) + else: + freqtrade.strategy.stop_loss_reached = MagicMock(return_value=SellCheckTuple( + sell_type=SellType.NONE)) freqtrade.enter_positions() trade = Trade.query.first() trade.update(limit_buy_order) freqtrade.wallets.update() patch_get_signal(freqtrade, value=(False, True, None)) - assert freqtrade.handle_trade(trade) is False + assert freqtrade.handle_trade(trade) is handle_first - freqtrade.strategy.sell_profit_offset = 0.0 - assert freqtrade.handle_trade(trade) is True + if handle_second: + freqtrade.strategy.sell_profit_offset = 0.0 + assert freqtrade.handle_trade(trade) is True - assert trade.sell_reason == SellType.SELL_SIGNAL.value - - -def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, limit_buy_order_open, - fee, mocker) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=MagicMock(return_value={ - 'bid': 0.00002172, - 'ask': 0.00002173, - 'last': 0.00002172 - }), - create_order=MagicMock(side_effect=[ - limit_buy_order_open, - {'id': 1234553382}, - ]), - get_fee=fee, - ) - default_conf.update({ - 'use_sell_signal': True, - 'sell_profit_only': False, - }) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.enter_positions() - - trade = Trade.query.first() - trade.update(limit_buy_order) - freqtrade.wallets.update() - patch_get_signal(freqtrade, value=(False, True, None)) - assert freqtrade.handle_trade(trade) is True - assert trade.sell_reason == SellType.SELL_SIGNAL.value - - -def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, limit_buy_order_open, - fee, mocker) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=MagicMock(return_value={ - 'bid': 0.00000172, - 'ask': 0.00000173, - 'last': 0.00000172 - }), - create_order=MagicMock(side_effect=[ - limit_buy_order_open, - {'id': 1234553382}, - ]), - get_fee=fee, - ) - default_conf.update({ - 'use_sell_signal': True, - 'sell_profit_only': True, - }) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - freqtrade.strategy.stop_loss_reached = MagicMock(return_value=SellCheckTuple( - sell_type=SellType.NONE)) - freqtrade.enter_positions() - - trade = Trade.query.first() - trade.update(limit_buy_order) - patch_get_signal(freqtrade, value=(False, True, None)) - assert freqtrade.handle_trade(trade) is False - - -def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, limit_buy_order_open, - fee, mocker) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=MagicMock(return_value={ - 'bid': 0.0000172, - 'ask': 0.0000173, - 'last': 0.0000172 - }), - create_order=MagicMock(side_effect=[ - limit_buy_order_open, - {'id': 1234553382}, - ]), - get_fee=fee, - ) - default_conf.update({ - 'use_sell_signal': True, - 'sell_profit_only': False, - }) - - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - - freqtrade.enter_positions() - - trade = Trade.query.first() - trade.update(limit_buy_order) - freqtrade.wallets.update() - patch_get_signal(freqtrade, value=(False, True, None)) - assert freqtrade.handle_trade(trade) is True - assert trade.sell_reason == SellType.SELL_SIGNAL.value + assert trade.sell_reason == sell_type def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_open, @@ -3306,11 +3219,15 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_ assert trade.amount != amnt -def test__safe_exit_amount(default_conf, fee, caplog, mocker): +@pytest.mark.parametrize('amount_wallet,has_err', [ + (95.29, False), + (91.29, True) +]) +def test__safe_exit_amount(default_conf, fee, caplog, mocker, amount_wallet, has_err): patch_RPCManager(mocker) patch_exchange(mocker) amount = 95.33 - amount_wallet = 95.29 + amount_wallet = amount_wallet mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=amount_wallet)) wallet_update = mocker.patch('freqtrade.wallets.Wallets.update') trade = Trade( @@ -3324,37 +3241,19 @@ def test__safe_exit_amount(default_conf, fee, caplog, mocker): ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - - wallet_update.reset_mock() - assert freqtrade._safe_exit_amount(trade.pair, trade.amount) == amount_wallet - assert log_has_re(r'.*Falling back to wallet-amount.', caplog) - assert wallet_update.call_count == 1 - caplog.clear() - wallet_update.reset_mock() - assert freqtrade._safe_exit_amount(trade.pair, amount_wallet) == amount_wallet - assert not log_has_re(r'.*Falling back to wallet-amount.', caplog) - assert wallet_update.call_count == 1 - - -def test__safe_exit_amount_error(default_conf, fee, caplog, mocker): - patch_RPCManager(mocker) - patch_exchange(mocker) - amount = 95.33 - amount_wallet = 91.29 - mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=amount_wallet)) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - open_rate=0.245441, - open_order_id="123456", - fee_open=fee.return_value, - fee_close=fee.return_value, - ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - with pytest.raises(DependencyException, match=r"Not enough amount to sell."): - assert freqtrade._safe_exit_amount(trade.pair, trade.amount) + if has_err: + with pytest.raises(DependencyException, match=r"Not enough amount to sell."): + assert freqtrade._safe_exit_amount(trade.pair, trade.amount) + else: + wallet_update.reset_mock() + assert freqtrade._safe_exit_amount(trade.pair, trade.amount) == amount_wallet + assert log_has_re(r'.*Falling back to wallet-amount.', caplog) + assert wallet_update.call_count == 1 + caplog.clear() + wallet_update.reset_mock() + assert freqtrade._safe_exit_amount(trade.pair, amount_wallet) == amount_wallet + assert not log_has_re(r'.*Falling back to wallet-amount.', caplog) + assert wallet_update.call_count == 1 def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplog) -> None: @@ -4131,50 +4030,37 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_o assert trade is None -def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: +@pytest.mark.parametrize('exception_thrown,ask,last,order_book_top,order_book', [ + (False, 0.045, 0.046, 2, None), + (True, 0.042, 0.046, 1, {'bids': [[]], 'asks': [[]]}) +]) +def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, exception_thrown, + ask, last, order_book_top, order_book, caplog) -> None: """ - test if function get_rate will return the order book price - instead of the ask rate + test if function get_rate will return the order book price instead of the ask rate """ patch_exchange(mocker) - ticker_mock = MagicMock(return_value={'ask': 0.045, 'last': 0.046}) + ticker_mock = MagicMock(return_value={'ask': ask, 'last': last}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_l2_order_book=order_book_l2, + fetch_l2_order_book=MagicMock(return_value=order_book) if order_book else order_book_l2, fetch_ticker=ticker_mock, - ) default_conf['exchange']['name'] = 'binance' default_conf['bid_strategy']['use_order_book'] = True - default_conf['bid_strategy']['order_book_top'] = 2 + default_conf['bid_strategy']['order_book_top'] = order_book_top default_conf['bid_strategy']['ask_last_balance'] = 0 default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) - assert freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") == 0.043935 - assert ticker_mock.call_count == 0 - - -def test_order_book_bid_strategy_exception(mocker, default_conf, caplog) -> None: - patch_exchange(mocker) - ticker_mock = MagicMock(return_value={'ask': 0.042, 'last': 0.046}) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_l2_order_book=MagicMock(return_value={'bids': [[]], 'asks': [[]]}), - fetch_ticker=ticker_mock, - - ) - default_conf['exchange']['name'] = 'binance' - default_conf['bid_strategy']['use_order_book'] = True - default_conf['bid_strategy']['order_book_top'] = 1 - default_conf['bid_strategy']['ask_last_balance'] = 0 - default_conf['telegram']['enabled'] = False - - freqtrade = FreqtradeBot(default_conf) - # orderbook shall be used even if tickers would be lower. - with pytest.raises(PricingError): - freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") - assert log_has_re(r'Buy Price at location 1 from orderbook could not be determined.', caplog) + if exception_thrown: + with pytest.raises(PricingError): + freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") + assert log_has_re( + r'Buy Price at location 1 from orderbook could not be determined.', caplog) + else: + assert freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") == 0.043935 + assert ticker_mock.call_count == 0 def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: From 42a2fdc1c520c97b27cf937e25aa7bd85a581a21 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 18 Sep 2021 03:01:08 -0600 Subject: [PATCH 107/239] parametrized test_order_dict --- tests/test_freqtradebot.py | 47 ++++++++------------------------------ 1 file changed, 10 insertions(+), 37 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 5268d40ec..8e8ae4da2 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -78,11 +78,15 @@ def test_bot_cleanup(mocker, default_conf, caplog) -> None: assert coo_mock.call_count == 1 -def test_order_dict_dry_run(default_conf, mocker, caplog) -> None: +@pytest.mark.parametrize('runmode', [ + RunMode.DRY_RUN, + RunMode.LIVE +]) +def test_order_dict(default_conf, mocker, runmode, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) conf = default_conf.copy() - conf['runmode'] = RunMode.DRY_RUN + conf['runmode'] = runmode conf['order_types'] = { 'buy': 'market', 'sell': 'limit', @@ -92,45 +96,14 @@ def test_order_dict_dry_run(default_conf, mocker, caplog) -> None: conf['bid_strategy']['price_side'] = 'ask' freqtrade = FreqtradeBot(conf) + if runmode == RunMode.LIVE: + assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) assert freqtrade.strategy.order_types['stoploss_on_exchange'] caplog.clear() # is left untouched conf = default_conf.copy() - conf['runmode'] = RunMode.DRY_RUN - conf['order_types'] = { - 'buy': 'market', - 'sell': 'limit', - 'stoploss': 'limit', - 'stoploss_on_exchange': False, - } - freqtrade = FreqtradeBot(conf) - assert not freqtrade.strategy.order_types['stoploss_on_exchange'] - assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) - - -def test_order_dict_live(default_conf, mocker, caplog) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - - conf = default_conf.copy() - conf['runmode'] = RunMode.LIVE - conf['order_types'] = { - 'buy': 'market', - 'sell': 'limit', - 'stoploss': 'limit', - 'stoploss_on_exchange': True, - } - conf['bid_strategy']['price_side'] = 'ask' - - freqtrade = FreqtradeBot(conf) - assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) - assert freqtrade.strategy.order_types['stoploss_on_exchange'] - - caplog.clear() - # is left untouched - conf = default_conf.copy() - conf['runmode'] = RunMode.LIVE + conf['runmode'] = runmode conf['order_types'] = { 'buy': 'market', 'sell': 'limit', @@ -3139,7 +3112,7 @@ def test_execute_trade_exit_insufficient_funds_error(default_conf, ticker, fee, # Disable loss (False, 0.00000172, 0.00000173, True, False, SellType.SELL_SIGNAL.value), ]) -def test_sell_profit_only_enable_profit( +def test_sell_profit_only( default_conf, limit_buy_order, limit_buy_order_open, fee, mocker, profit_only, bid, ask, handle_first, handle_second, sell_type) -> None: patch_RPCManager(mocker) From 518a59ad41daa50c51deb8d7e70e75b3b9a8bad0 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 18 Sep 2021 03:20:00 -0600 Subject: [PATCH 108/239] parametrized test_edge_overrides_stoploss --- tests/test_freqtradebot.py | 60 ++++++++++---------------------------- 1 file changed, 15 insertions(+), 45 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 8e8ae4da2..debe2bee0 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -192,8 +192,14 @@ def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: 'LTC/BTC', freqtrade.edge) == (999.9 * 0.5 * 0.01) / 0.21 -def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf) -> None: - +@pytest.mark.parametrize('buy_price_mult,ignore_strat_sl', [ + # Override stoploss + (0.79, False), + # Override strategy stoploss + (0.85, True) +]) +def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, + buy_price_mult, ignore_strat_sl, edge_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) patch_edge(mocker) @@ -207,9 +213,9 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': buy_price * 0.79, - 'ask': buy_price * 0.79, - 'last': buy_price * 0.79 + 'bid': buy_price * buy_price_mult, + 'ask': buy_price * buy_price_mult, + 'last': buy_price * buy_price_mult, }), get_fee=fee, ) @@ -226,46 +232,10 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf ############################################# # stoploss shoud be hit - assert freqtrade.handle_trade(trade) is True - assert log_has('Executing Sell for NEO/BTC. Reason: stop_loss', caplog) - assert trade.sell_reason == SellType.STOP_LOSS.value - - -def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, - mocker, edge_conf) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - patch_edge(mocker) - edge_conf['max_open_trades'] = float('inf') - - # Strategy stoploss is -0.1 but Edge imposes a stoploss at -0.2 - # Thus, if price falls 15%, stoploss should not be triggered - # - # mocking the ticker: price is falling ... - buy_price = limit_buy_order['price'] - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=MagicMock(return_value={ - 'bid': buy_price * 0.85, - 'ask': buy_price * 0.85, - 'last': buy_price * 0.85 - }), - get_fee=fee, - ) - ############################################# - - # Create a trade with "limit_buy_order" price - freqtrade = FreqtradeBot(edge_conf) - freqtrade.active_pair_whitelist = ['NEO/BTC'] - patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.enter_positions() - trade = Trade.query.first() - trade.update(limit_buy_order) - ############################################# - - # stoploss shoud not be hit - assert freqtrade.handle_trade(trade) is False + assert freqtrade.handle_trade(trade) is not ignore_strat_sl + if not ignore_strat_sl: + assert log_has('Executing Sell for NEO/BTC. Reason: stop_loss', caplog) + assert trade.sell_reason == SellType.STOP_LOSS.value def test_total_open_trades_stakes(mocker, default_conf, ticker, fee) -> None: From 713e7819f7e534a36984f9f4c76a11bbfd948896 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sat, 18 Sep 2021 15:27:58 +0300 Subject: [PATCH 109/239] [SQUASH] Remove mypy import. --- freqtrade/strategy/informative_decorator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/informative_decorator.py b/freqtrade/strategy/informative_decorator.py index c8ebf5989..4c5f21108 100644 --- a/freqtrade/strategy/informative_decorator.py +++ b/freqtrade/strategy/informative_decorator.py @@ -1,6 +1,5 @@ from typing import Any, Callable, NamedTuple, Optional, Union -from mypy_extensions import KwArg from pandas import DataFrame from freqtrade.exceptions import OperationalException @@ -13,12 +12,12 @@ PopulateIndicators = Callable[[Any, DataFrame, dict], DataFrame] class InformativeData(NamedTuple): asset: Optional[str] timeframe: str - fmt: Union[str, Callable[[KwArg(str)], str], None] + fmt: Union[str, Callable[[Any], str], None] ffill: bool def informative(timeframe: str, asset: str = '', - fmt: Optional[Union[str, Callable[[KwArg(str)], str]]] = None, + fmt: Optional[Union[str, Callable[[Any], str]]] = None, ffill: bool = True) -> Callable[[PopulateIndicators], PopulateIndicators]: """ A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to From eab7f8f6944e77fa7ae69bf99fe885aa76aea5a5 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sat, 18 Sep 2021 15:44:21 +0300 Subject: [PATCH 110/239] [SQUASH] Doh. --- tests/strategy/test_strategy_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 9132382fa..a01b55050 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -5,8 +5,8 @@ import pandas as pd import pytest from freqtrade.data.dataprovider import DataProvider -from freqtrade.strategy import (merge_informative_pair, stoploss_from_open, timeframe_to_minutes, - stoploss_from_absolute) +from freqtrade.strategy import (merge_informative_pair, stoploss_from_absolute, stoploss_from_open, + timeframe_to_minutes) def generate_test_data(timeframe: str, size: int): From c1895a0fc230f14e9de79a3042d77d3379a61c1f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Sep 2021 15:30:33 +0200 Subject: [PATCH 111/239] Remove warning related to legacy hyperopt --- docs/strategy-customization.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 2b22dd274..725252b30 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -801,9 +801,6 @@ for more information. will overwrite previously defined method and not produce any errors due to limitations of Python programming language. In such cases you will find that indicators created in earlier-defined methods are not available in the dataframe. Carefully review method names and make sure they are unique! -!!! Warning - When using a legacy hyperopt implementation informative pairs defined with a decorator will not be executed. Please update your strategy if necessary. - ## Additional data (Wallets) The strategy provides access to the `Wallets` object. This contains the current balances on the exchange. From ec9dbc550e66c2694bfe32a43c3442952d8b685c Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 18 Sep 2021 19:19:53 -0600 Subject: [PATCH 112/239] parametrized test_create_trade_minimal_amount --- tests/test_freqtradebot.py | 93 +++++++++++--------------------------- 1 file changed, 26 insertions(+), 67 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index debe2bee0..209c8a2a2 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -319,8 +319,16 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, freqtrade.create_trade('ETH/BTC') -def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order_open, - fee, mocker) -> None: +@pytest.mark.parametrize('stake_amount,create,amount_enough,max_open_trades', [ + (0.0005, True, True, 99), + (0.000000005, True, False, 99), + (0, False, True, 99), + (UNLIMITED_STAKE_AMOUNT, False, True, 0), +]) +def test_create_trade_minimal_amount( + default_conf, ticker, limit_buy_order_open, fee, mocker, + stake_amount, create, amount_enough, max_open_trades, caplog +) -> None: patch_RPCManager(mocker) patch_exchange(mocker) buy_mock = MagicMock(return_value=limit_buy_order_open) @@ -330,74 +338,25 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order_open, create_order=buy_mock, get_fee=fee, ) - default_conf['stake_amount'] = 0.0005 + default_conf['max_open_trades'] = max_open_trades freqtrade = FreqtradeBot(default_conf) + freqtrade.config['stake_amount'] = stake_amount patch_get_signal(freqtrade) - freqtrade.create_trade('ETH/BTC') - rate, amount = buy_mock.call_args[1]['rate'], buy_mock.call_args[1]['amount'] - assert rate * amount <= default_conf['stake_amount'] - - -def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order_open, - fee, mocker, caplog) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - buy_mock = MagicMock(return_value=limit_buy_order_open) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, - create_order=buy_mock, - get_fee=fee, - ) - - freqtrade = FreqtradeBot(default_conf) - freqtrade.config['stake_amount'] = 0.000000005 - - patch_get_signal(freqtrade) - - assert freqtrade.create_trade('ETH/BTC') - assert log_has_re(r"Stake amount for pair .* is too small.*", caplog) - - -def test_create_trade_zero_stake_amount(default_conf, ticker, limit_buy_order_open, - fee, mocker) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - buy_mock = MagicMock(return_value=limit_buy_order_open) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, - create_order=buy_mock, - get_fee=fee, - ) - - freqtrade = FreqtradeBot(default_conf) - freqtrade.config['stake_amount'] = 0 - - patch_get_signal(freqtrade) - - assert not freqtrade.create_trade('ETH/BTC') - - -def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order_open, - fee, mocker) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, - create_order=MagicMock(return_value=limit_buy_order_open), - get_fee=fee, - ) - default_conf['max_open_trades'] = 0 - default_conf['stake_amount'] = UNLIMITED_STAKE_AMOUNT - - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - - assert not freqtrade.create_trade('ETH/BTC') - assert freqtrade.wallets.get_trade_stake_amount('ETH/BTC', freqtrade.edge) == 0 + if create: + assert freqtrade.create_trade('ETH/BTC') + if amount_enough: + rate, amount = buy_mock.call_args[1]['rate'], buy_mock.call_args[1]['amount'] + assert rate * amount <= default_conf['stake_amount'] + else: + assert log_has_re( + r"Stake amount for pair .* is too small.*", + caplog + ) + else: + assert not freqtrade.create_trade('ETH/BTC') + if not max_open_trades: + assert freqtrade.wallets.get_trade_stake_amount('ETH/BTC', freqtrade.edge) == 0 def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_open, fee, From cee4ed541b8e4bec16ce8a23598531fb4d64ec57 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 18 Sep 2021 19:46:46 -0600 Subject: [PATCH 113/239] parametrized test_update_trade_state_withorderdict --- tests/test_freqtradebot.py | 104 +++++++++++++------------------------ 1 file changed, 35 insertions(+), 69 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 209c8a2a2..72d1f6150 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -359,8 +359,12 @@ def test_create_trade_minimal_amount( assert freqtrade.wallets.get_trade_stake_amount('ETH/BTC', freqtrade.edge) == 0 +@pytest.mark.parametrize('whitelist,positions', [ + (["ETH/BTC"], 1), # No pairs left + ([], 0), # No pairs in whitelist +]) def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_open, fee, - mocker, caplog) -> None: + whitelist, positions, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -369,36 +373,20 @@ def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_ope create_order=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) - - default_conf['exchange']['pair_whitelist'] = ["ETH/BTC"] + default_conf['exchange']['pair_whitelist'] = whitelist freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) n = freqtrade.enter_positions() - assert n == 1 - assert not log_has_re(r"No currency pair in active pair whitelist.*", caplog) - n = freqtrade.enter_positions() - assert n == 0 - assert log_has_re(r"No currency pair in active pair whitelist.*", caplog) - - -def test_enter_positions_no_pairs_in_whitelist(default_conf, ticker, limit_buy_order, fee, - mocker, caplog) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, - create_order=MagicMock(return_value={'id': limit_buy_order['id']}), - get_fee=fee, - ) - default_conf['exchange']['pair_whitelist'] = [] - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - - n = freqtrade.enter_positions() - assert n == 0 - assert log_has("Active pair whitelist is empty.", caplog) + assert n == positions + if positions: + assert not log_has_re(r"No currency pair in active pair whitelist.*", caplog) + n = freqtrade.enter_positions() + assert n == 0 + assert log_has_re(r"No currency pair in active pair whitelist.*", caplog) + else: + assert n == 0 + assert log_has("Active pair whitelist is empty.", caplog) @pytest.mark.usefixtures("init_persistence") @@ -1555,30 +1543,27 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, stop_price=0.00002346 * 0.99) -def test_enter_positions(mocker, default_conf, caplog) -> None: +@pytest.mark.parametrize('return_value,side_effect,log_message', [ + (False, None, 'Found no buy signals for whitelisted currencies. Trying again...'), + (None, DependencyException, 'Unable to create trade for ETH/BTC: ') +]) +def test_enter_positions(mocker, default_conf, return_value, side_effect, + log_message, caplog) -> None: caplog.set_level(logging.DEBUG) freqtrade = get_patched_freqtradebot(mocker, default_conf) - mock_ct = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trade', - MagicMock(return_value=False)) - n = freqtrade.enter_positions() - assert n == 0 - assert log_has('Found no buy signals for whitelisted currencies. Trying again...', caplog) - # create_trade should be called once for every pair in the whitelist. - assert mock_ct.call_count == len(default_conf['exchange']['pair_whitelist']) - - -def test_enter_positions_exception(mocker, default_conf, caplog) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mock_ct = mocker.patch( 'freqtrade.freqtradebot.FreqtradeBot.create_trade', - MagicMock(side_effect=DependencyException) + MagicMock( + return_value=return_value, + side_effect=side_effect + ) ) n = freqtrade.enter_positions() assert n == 0 + assert log_has(log_message, caplog) + # create_trade should be called once for every pair in the whitelist. assert mock_ct.call_count == len(default_conf['exchange']['pair_whitelist']) - assert log_has('Unable to create trade for ETH/BTC: ', caplog) def test_exit_positions(mocker, default_conf, limit_buy_order, caplog) -> None: @@ -1672,8 +1657,13 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No assert log_has_re('Found open order for.*', caplog) +@pytest.mark.parametrize('initial_amount,has_rounding_fee', [ + (90.99181073 + 1e-14, True), + (8.0, False) +]) def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_buy_order, fee, - mocker): + mocker, initial_amount, has_rounding_fee, caplog): + trades_for_order[0]['amount'] = initial_amount mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) # fetch_order should not be called!! mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError)) @@ -1694,32 +1684,8 @@ def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_ freqtrade.update_trade_state(trade, '123456', limit_buy_order) assert trade.amount != amount assert trade.amount == limit_buy_order['amount'] - - -def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_order, fee, - limit_buy_order, mocker, caplog): - trades_for_order[0]['amount'] = limit_buy_order['amount'] + 1e-14 - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - # fetch_order should not be called!! - mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError)) - patch_exchange(mocker) - amount = sum(x['amount'] for x in trades_for_order) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - open_rate=0.245441, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_order_id='123456', - is_open=True, - open_date=arrow.utcnow().datetime, - ) - freqtrade.update_trade_state(trade, '123456', limit_buy_order) - assert trade.amount != amount - assert trade.amount == limit_buy_order['amount'] - assert log_has_re(r'Applying fee on amount for .*', caplog) + if has_rounding_fee: + assert log_has_re(r'Applying fee on amount for .*', caplog) def test_update_trade_state_exception(mocker, default_conf, From ab88217186afa23718421ad67867a9d624024878 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Sep 2021 13:16:30 +0200 Subject: [PATCH 114/239] Improve /balance output to include starting balance and percentual change closes #5503 --- freqtrade/rpc/api_server/api_schemas.py | 7 +++++++ freqtrade/rpc/rpc.py | 24 ++++++++++++++++++++---- freqtrade/rpc/telegram.py | 23 +++++++++++++++-------- tests/rpc/test_rpc_apiserver.py | 14 ++++++++++---- tests/rpc/test_rpc_telegram.py | 2 ++ 5 files changed, 54 insertions(+), 16 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 3adbebc16..eb6082087 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -46,6 +46,13 @@ class Balances(BaseModel): value: float stake: str note: str + starting_capital: float + starting_capital_ratio: float + starting_capital_pct: float + starting_capital_fiat: float + starting_capital_fiat_ratio: float + starting_capital_fiat_pct: float + class Count(BaseModel): diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index b7b1fe603..12a444b80 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -458,6 +458,9 @@ class RPC: raise RPCException('Error getting current tickers.') self._freqtrade.wallets.update(require_update=False) + starting_capital = self._freqtrade.wallets.get_starting_balance() + starting_capital_fiat = self._fiat_converter.convert_amount( + starting_capital, stake_currency, fiat_display_currency) if self._fiat_converter else 0 for coin, balance in self._freqtrade.wallets.get_all_balances().items(): if not balance.total: @@ -493,15 +496,28 @@ class RPC: else: raise RPCException('All balances are zero.') - symbol = fiat_display_currency - value = self._fiat_converter.convert_amount(total, stake_currency, - symbol) if self._fiat_converter else 0 + value = self._fiat_converter.convert_amount( + total, stake_currency, fiat_display_currency) if self._fiat_converter else 0 + + starting_capital_ratio = 0.0 + starting_capital_fiat_ratio = 0.0 + if starting_capital: + starting_capital_ratio = (total / starting_capital) - 1 + if starting_capital_fiat: + starting_capital_fiat_ratio = (value / starting_capital_fiat) - 1 + return { 'currencies': output, 'total': total, - 'symbol': symbol, + 'symbol': fiat_display_currency, 'value': value, 'stake': stake_currency, + 'starting_capital': starting_capital, + 'starting_capital_ratio': starting_capital_ratio, + 'starting_capital_pct': round(starting_capital_ratio * 100, 2), + 'starting_capital_fiat': starting_capital_fiat, + 'starting_capital_fiat_ratio': starting_capital_fiat_ratio, + 'starting_capital_fiat_pct': round(starting_capital_fiat_ratio * 100, 2), 'note': 'Simulated balances' if self._freqtrade.config['dry_run'] else '' } diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a988d2b60..ffb7385da 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -603,12 +603,15 @@ class Telegram(RPCHandler): output = '' if self._config['dry_run']: - output += ( - f"*Warning:* Simulated balances in Dry Mode.\n" - "This mode is still experimental!\n" - "Starting capital: " - f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n" - ) + output += "*Warning:* Simulated balances in Dry Mode.\n" + + output += ("Starting capital: " + f"`{result['starting_capital']}` {self._config['stake_currency']}" + ) + output += (f" `{result['starting_capital_fiat']}` " + f"{self._config['fiat_display_currency']}.\n" + ) if result['starting_capital_fiat'] > 0 else '.\n' + total_dust_balance = 0 total_dust_currencies = 0 for curr in result['currencies']: @@ -641,9 +644,13 @@ class Telegram(RPCHandler): f"{round_coin_value(total_dust_balance, result['stake'], False)}`\n") output += ("\n*Estimated Value*:\n" - f"\t`{result['stake']}: {result['total']: .8f}`\n" + f"\t`{result['stake']}: " + f"{round_coin_value(result['total'], result['stake'], False)}`" + f" `({result['starting_capital_pct']}%)`\n" f"\t`{result['symbol']}: " - f"{round_coin_value(result['value'], result['symbol'], False)}`\n") + f"{round_coin_value(result['value'], result['symbol'], False)}`" + f" `({result['starting_capital_fiat_pct']}%)`\n" + ) self._send_msg(output, reload_able=True, callback_path="update_balance", query=update.callback_query) except RPCException as e: diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 43eb70938..7c98b2df7 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -422,20 +422,22 @@ def test_api_stopbuy(botclient): assert ftbot.config['max_open_trades'] == 0 -def test_api_balance(botclient, mocker, rpc_balance): +def test_api_balance(botclient, mocker, rpc_balance, tickers): ftbot, client = botclient ftbot.config['dry_run'] = False mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value=rpc_balance) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) mocker.patch('freqtrade.exchange.Exchange.get_valid_pair_combination', side_effect=lambda a, b: f"{a}/{b}") ftbot.wallets.update() rc = client_get(client, f"{BASE_URI}/balance") assert_response(rc) - assert "currencies" in rc.json() - assert len(rc.json()["currencies"]) == 5 - assert rc.json()['currencies'][0] == { + response = rc.json() + assert "currencies" in response + assert len(response["currencies"]) == 5 + assert response['currencies'][0] == { 'currency': 'BTC', 'free': 12.0, 'balance': 12.0, @@ -443,6 +445,10 @@ def test_api_balance(botclient, mocker, rpc_balance): 'est_stake': 12.0, 'stake': 'BTC', } + assert 'starting_capital' in response + assert 'starting_capital_fiat' in response + assert 'starting_capital_pct' in response + assert 'starting_capital_ratio' in response def test_api_count(botclient, mocker, ticker, fee, markets): diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 2013dad7d..21f1cd000 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -576,6 +576,8 @@ def test_balance_handle_too_large_response(default_conf, update, mocker) -> None 'total': 100.0, 'symbol': 100.0, 'value': 1000.0, + 'starting_capital': 1000, + 'starting_capital_fiat': 1000, }) telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) From ec03531771904a3a20dc31e4c64d0f13156fdcce Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Sep 2021 13:29:09 +0200 Subject: [PATCH 115/239] Improve naming of variables --- freqtrade/rpc/api_server/api_schemas.py | 1 - freqtrade/rpc/rpc.py | 15 ++++++--------- freqtrade/rpc/telegram.py | 3 +-- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index eb6082087..46187f571 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -54,7 +54,6 @@ class Balances(BaseModel): starting_capital_fiat_pct: float - class Count(BaseModel): current: int max: int diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 12a444b80..f6599b429 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -459,7 +459,7 @@ class RPC: self._freqtrade.wallets.update(require_update=False) starting_capital = self._freqtrade.wallets.get_starting_balance() - starting_capital_fiat = self._fiat_converter.convert_amount( + starting_cap_fiat = self._fiat_converter.convert_amount( starting_capital, stake_currency, fiat_display_currency) if self._fiat_converter else 0 for coin, balance in self._freqtrade.wallets.get_all_balances().items(): @@ -500,11 +500,8 @@ class RPC: total, stake_currency, fiat_display_currency) if self._fiat_converter else 0 starting_capital_ratio = 0.0 - starting_capital_fiat_ratio = 0.0 - if starting_capital: - starting_capital_ratio = (total / starting_capital) - 1 - if starting_capital_fiat: - starting_capital_fiat_ratio = (value / starting_capital_fiat) - 1 + starting_capital_ratio = (total / starting_capital) - 1 if starting_capital else 0.0 + starting_cap_fiat_ratio = (value / starting_cap_fiat) - 1 if starting_cap_fiat else 0.0 return { 'currencies': output, @@ -515,9 +512,9 @@ class RPC: 'starting_capital': starting_capital, 'starting_capital_ratio': starting_capital_ratio, 'starting_capital_pct': round(starting_capital_ratio * 100, 2), - 'starting_capital_fiat': starting_capital_fiat, - 'starting_capital_fiat_ratio': starting_capital_fiat_ratio, - 'starting_capital_fiat_pct': round(starting_capital_fiat_ratio * 100, 2), + 'starting_capital_fiat': starting_cap_fiat, + 'starting_capital_fiat_ratio': starting_cap_fiat_ratio, + 'starting_capital_fiat_pct': round(starting_cap_fiat_ratio * 100, 2), 'note': 'Simulated balances' if self._freqtrade.config['dry_run'] else '' } diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ffb7385da..19c58b63d 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -649,8 +649,7 @@ class Telegram(RPCHandler): f" `({result['starting_capital_pct']}%)`\n" f"\t`{result['symbol']}: " f"{round_coin_value(result['value'], result['symbol'], False)}`" - f" `({result['starting_capital_fiat_pct']}%)`\n" - ) + f" `({result['starting_capital_fiat_pct']}%)`\n") self._send_msg(output, reload_able=True, callback_path="update_balance", query=update.callback_query) except RPCException as e: From 879bf47b32ff692e35f2509665b1ef9899eb3621 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Sep 2021 19:25:36 +0200 Subject: [PATCH 116/239] Refactor telegram.py to simplify send_msg --- freqtrade/rpc/telegram.py | 51 +++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 19c58b63d..898446ea0 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -260,29 +260,7 @@ class Telegram(RPCHandler): return message - def send_msg(self, msg: Dict[str, Any]) -> None: - """ Send a message to telegram channel """ - - default_noti = 'on' - - msg_type = msg['type'] - noti = '' - if msg_type == RPCMessageType.SELL: - sell_noti = self._config['telegram'] \ - .get('notification_settings', {}).get(str(msg_type), {}) - # For backward compatibility sell still can be string - if isinstance(sell_noti, str): - noti = sell_noti - else: - noti = sell_noti.get(str(msg['sell_reason']), default_noti) - else: - noti = self._config['telegram'] \ - .get('notification_settings', {}).get(str(msg_type), default_noti) - - if noti == 'off': - logger.info(f"Notification '{msg_type}' not sent.") - # Notification disabled - return + def compose_message(self, msg: Dict[str, Any], msg_type: RPCMessageType) -> str: if msg_type == RPCMessageType.BUY: message = self._format_buy_msg(msg) @@ -315,6 +293,33 @@ class Telegram(RPCHandler): else: raise NotImplementedError('Unknown message type: {}'.format(msg_type)) + return message + + def send_msg(self, msg: Dict[str, Any]) -> None: + """ Send a message to telegram channel """ + + default_noti = 'on' + + msg_type = msg['type'] + noti = '' + if msg_type == RPCMessageType.SELL: + sell_noti = self._config['telegram'] \ + .get('notification_settings', {}).get(str(msg_type), {}) + # For backward compatibility sell still can be string + if isinstance(sell_noti, str): + noti = sell_noti + else: + noti = sell_noti.get(str(msg['sell_reason']), default_noti) + else: + noti = self._config['telegram'] \ + .get('notification_settings', {}).get(str(msg_type), default_noti) + + if noti == 'off': + logger.info(f"Notification '{msg_type}' not sent.") + # Notification disabled + return + + message = self.compose_message(msg, msg_type) self._send_msg(message, disable_notification=(noti == 'silent')) From 1da091dea3006b67e2400cf1390981354626f4fb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Sep 2021 19:41:19 +0200 Subject: [PATCH 117/239] ProtectionManager should return the lock just created --- freqtrade/persistence/pairlock_middleware.py | 4 +++- freqtrade/plugins/protectionmanager.py | 23 ++++++++++---------- tests/plugins/test_protections.py | 4 ++-- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index af904f693..8662fc36d 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -30,7 +30,8 @@ class PairLocks(): PairLocks.locks = [] @staticmethod - def lock_pair(pair: str, until: datetime, reason: str = None, *, now: datetime = None) -> None: + def lock_pair(pair: str, until: datetime, reason: str = None, *, + now: datetime = None) -> PairLock: """ Create PairLock from now to "until". Uses database by default, unless PairLocks.use_db is set to False, @@ -52,6 +53,7 @@ class PairLocks(): PairLock.query.session.commit() else: PairLocks.locks.append(lock) + return lock @staticmethod def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]: diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index f33e5b4bc..2510d6fee 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from typing import Dict, List, Optional from freqtrade.persistence import PairLocks +from freqtrade.persistence.models import PairLock from freqtrade.plugins.protections import IProtection from freqtrade.resolvers import ProtectionResolver @@ -43,30 +44,28 @@ class ProtectionManager(): """ return [{p.name: p.short_desc()} for p in self._protection_handlers] - def global_stop(self, now: Optional[datetime] = None) -> bool: + def global_stop(self, now: Optional[datetime] = None) -> Optional[PairLock]: if not now: now = datetime.now(timezone.utc) - result = False + result = None for protection_handler in self._protection_handlers: if protection_handler.has_global_stop: - result, until, reason = protection_handler.global_stop(now) + lock, until, reason = protection_handler.global_stop(now) # Early stopping - first positive result blocks further trades - if result and until: + if lock and until: if not PairLocks.is_global_lock(until): - PairLocks.lock_pair('*', until, reason, now=now) - result = True + result = PairLocks.lock_pair('*', until, reason, now=now) return result - def stop_per_pair(self, pair, now: Optional[datetime] = None) -> bool: + def stop_per_pair(self, pair, now: Optional[datetime] = None) -> Optional[PairLock]: if not now: now = datetime.now(timezone.utc) - result = False + result = None for protection_handler in self._protection_handlers: if protection_handler.has_local_stop: - result, until, reason = protection_handler.stop_per_pair(pair, now) - if result and until: + lock, until, reason = protection_handler.stop_per_pair(pair, now) + if lock and until: if not PairLocks.is_pair_locked(pair, until): - PairLocks.lock_pair(pair, until, reason, now=now) - result = True + result = PairLocks.lock_pair(pair, until, reason, now=now) return result diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index c0a9ae72a..a3cb29c9d 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -125,7 +125,7 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): # Test 5m after lock-period - this should try and relock the pair, but end-time # should be the previous end-time end_time = PairLocks.get_pair_longest_lock('*').lock_end_time + timedelta(minutes=5) - assert freqtrade.protections.global_stop(end_time) + freqtrade.protections.global_stop(end_time) assert not PairLocks.is_global_lock(end_time) @@ -182,7 +182,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair min_ago_open=180, min_ago_close=30, profit_rate=0.9, )) - assert freqtrade.protections.stop_per_pair(pair) + freqtrade.protections.stop_per_pair(pair) assert freqtrade.protections.global_stop() != only_per_pair assert PairLocks.is_pair_locked(pair) assert PairLocks.is_global_lock() != only_per_pair From c91a9a92f2ccecc5aac46f25a999c0b4a97b20a7 Mon Sep 17 00:00:00 2001 From: Bernhard Millauer Date: Mon, 20 Sep 2021 14:22:24 +0200 Subject: [PATCH 118/239] Add troubleshooting information The time in wsl docker container shifts over time. Added information how to fix this issue. --- docs/docker_quickstart.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 1fa229225..f4f8c366d 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -149,6 +149,20 @@ You'll then also need to modify the `docker-compose.yml` file and uncomment the You can then run `docker-compose build` to build the docker image, and run it using the commands described above. +### Troubleshooting + +### Docker on Windows + +* Error: `"Timestamp for this request is outside of the recvWindow."` + * The market api requests require a synchronized clock but the time in the docker container shifts a bit over time into the past. + To fix this issue temporarily you need to run `wsl --shutdown` and restart docker again (a popup on windows 10 will ask you to do so). + A permanent solution is either to host the docker container on a linux host or restart the wsl from time to time with the scheduler. + ``` + taskkill /IM "Docker Desktop.exe" /F + wsl --shutdown + start "" "C:\Program Files\Docker\Docker\Docker Desktop.exe" + ``` + ## Plotting with docker-compose Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your docker-compose.yml file. From a0fb43c6caca7d0b0f06cf9814d43f69341339b6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Sep 2021 19:12:59 +0200 Subject: [PATCH 119/239] Add pairlock-notification --- freqtrade/enums/rpcmessagetype.py | 1 + freqtrade/freqtradebot.py | 16 ++++++++++++++-- freqtrade/rpc/telegram.py | 5 +++++ tests/rpc/test_rpc_telegram.py | 14 ++++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/freqtrade/enums/rpcmessagetype.py b/freqtrade/enums/rpcmessagetype.py index 9c59f6108..889d67cf4 100644 --- a/freqtrade/enums/rpcmessagetype.py +++ b/freqtrade/enums/rpcmessagetype.py @@ -11,6 +11,7 @@ class RPCMessageType(Enum): SELL = 'sell' SELL_FILL = 'sell_fill' SELL_CANCEL = 'sell_cancel' + PROTECTION_TRIGGER = 'protection_trigger' def __repr__(self): return self.value diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1cb8988ff..451b5764a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1292,8 +1292,7 @@ class FreqtradeBot(LoggingMixin): if not trade.is_open: if not stoploss_order and not trade.open_order_id: self._notify_exit(trade, '', True) - self.protections.stop_per_pair(trade.pair) - self.protections.global_stop() + self.handle_protections(trade.pair) self.wallets.update() elif not trade.open_order_id: # Buy fill @@ -1301,6 +1300,19 @@ class FreqtradeBot(LoggingMixin): return False + def handle_protections(self, pair: str) -> None: + prot_trig = self.protections.stop_per_pair(pair) + if prot_trig: + msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } + msg.update(prot_trig.to_json()) + self.rpc.send_msg(msg) + + prot_trig_glb = self.protections.global_stop() + if prot_trig_glb: + msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } + msg.update(prot_trig_glb.to_json()) + self.rpc.send_msg(msg) + def apply_fee_conditional(self, trade: Trade, trade_base_currency: str, amount: float, fee_abs: float) -> float: """ diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 898446ea0..934a5182a 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -281,6 +281,11 @@ class Telegram(RPCHandler): "for {close_rate}.".format(**msg)) elif msg_type == RPCMessageType.SELL: message = self._format_sell_msg(msg) + elif msg_type == RPCMessageType.PROTECTION_TRIGGER: + message = ( + "*Protection* triggered due to {reason}. " + "{pair} will be locked until {lock_end_time}." + ).format(**msg) elif msg_type == RPCMessageType.STATUS: message = '*Status:* `{status}`'.format(**msg) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 21f1cd000..0675e4d5f 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1313,6 +1313,20 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None: 'Reason: cancelled due to timeout.') +def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> None: + + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) + time_machine.move_to("2021-09-01 05:00:00 +00:00") + lock = PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=6).datetime, 'randreason') + msg = { + 'type': RPCMessageType.PROTECTION_TRIGGER, + } + msg.update(lock.to_json()) + telegram.send_msg(msg) + assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " + "ETH/BTC will be locked until 2021-09-01 05:10:00.") + + def test_send_msg_buy_fill_notification(default_conf, mocker) -> None: default_conf['telegram']['notification_settings']['buy_fill'] = 'on' From dd0db7ee5db04fb98b542e9541d46768b9bb2c05 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Sep 2021 19:23:40 +0200 Subject: [PATCH 120/239] Split protection-notification into global and per-pair --- config_examples/config_full.example.json | 4 +++- docs/telegram-usage.md | 4 +++- freqtrade/enums/rpcmessagetype.py | 1 + freqtrade/freqtradebot.py | 2 +- freqtrade/rpc/telegram.py | 6 +++++- tests/rpc/test_rpc_telegram.py | 13 +++++++++++++ 6 files changed, 26 insertions(+), 4 deletions(-) diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index d0f3f0df6..c415d70b0 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -149,7 +149,9 @@ }, "sell_fill": "on", "buy_cancel": "on", - "sell_cancel": "on" + "sell_cancel": "on", + "protection_trigger": "off", + "protection_trigger_global": "on" }, "reload": true, "balance_dust_level": 0.01 diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index b020b00db..1e6fa9dae 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -93,7 +93,9 @@ Example configuration showing the different settings: "buy_cancel": "silent", "sell_cancel": "on", "buy_fill": "off", - "sell_fill": "off" + "sell_fill": "off", + "protection_trigger": "off", + "protection_trigger_global": "on" }, "reload": true, "balance_dust_level": 0.01 diff --git a/freqtrade/enums/rpcmessagetype.py b/freqtrade/enums/rpcmessagetype.py index 889d67cf4..4e3f693e5 100644 --- a/freqtrade/enums/rpcmessagetype.py +++ b/freqtrade/enums/rpcmessagetype.py @@ -12,6 +12,7 @@ class RPCMessageType(Enum): SELL_FILL = 'sell_fill' SELL_CANCEL = 'sell_cancel' PROTECTION_TRIGGER = 'protection_trigger' + PROTECTION_TRIGGER_GLOBAL = 'protection_trigger_global' def __repr__(self): return self.value diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 451b5764a..37be3173a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1309,7 +1309,7 @@ class FreqtradeBot(LoggingMixin): prot_trig_glb = self.protections.global_stop() if prot_trig_glb: - msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } + msg = {'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, } msg.update(prot_trig_glb.to_json()) self.rpc.send_msg(msg) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 934a5182a..0687f95a2 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -286,7 +286,11 @@ class Telegram(RPCHandler): "*Protection* triggered due to {reason}. " "{pair} will be locked until {lock_end_time}." ).format(**msg) - + elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL: + message = ( + "*Protection* triggered due to {reason}. " + "All pairs will be locked until {lock_end_time}." + ).format(**msg) elif msg_type == RPCMessageType.STATUS: message = '*Status:* `{status}`'.format(**msg) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 0675e4d5f..d37a74187 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1326,6 +1326,19 @@ def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " "ETH/BTC will be locked until 2021-09-01 05:10:00.") + msg_mock.reset_mock() + # Test global protection + + msg = { + 'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, + } + lock = PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=100).datetime, 'randreason') + msg.update(lock.to_json()) + telegram.send_msg(msg) + assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " + "All pairs will be locked until 2021-09-01 06:45:00.") + + def test_send_msg_buy_fill_notification(default_conf, mocker) -> None: From fd23ab3d647db7444d12129755ddf001e014dbcd Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Sep 2021 19:49:18 +0200 Subject: [PATCH 121/239] improve formatting, add tests --- freqtrade/rpc/telegram.py | 4 ++-- tests/rpc/test_rpc_telegram.py | 5 ++--- tests/test_freqtradebot.py | 23 +++++++++++++++++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 0687f95a2..059ba9c41 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -284,12 +284,12 @@ class Telegram(RPCHandler): elif msg_type == RPCMessageType.PROTECTION_TRIGGER: message = ( "*Protection* triggered due to {reason}. " - "{pair} will be locked until {lock_end_time}." + "`{pair}` will be locked until `{lock_end_time}`." ).format(**msg) elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL: message = ( "*Protection* triggered due to {reason}. " - "All pairs will be locked until {lock_end_time}." + "*All pairs* will be locked until `{lock_end_time}`." ).format(**msg) elif msg_type == RPCMessageType.STATUS: message = '*Status:* `{status}`'.format(**msg) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index d37a74187..8c285a76e 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1324,7 +1324,7 @@ def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> msg.update(lock.to_json()) telegram.send_msg(msg) assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " - "ETH/BTC will be locked until 2021-09-01 05:10:00.") + "`ETH/BTC` will be locked until `2021-09-01 05:10:00`.") msg_mock.reset_mock() # Test global protection @@ -1336,8 +1336,7 @@ def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> msg.update(lock.to_json()) telegram.send_msg(msg) assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " - "All pairs will be locked until 2021-09-01 06:45:00.") - + "*All pairs* will be locked until `2021-09-01 06:45:00`.") def test_send_msg_buy_fill_notification(default_conf, mocker) -> None: diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index eb3c77cc7..e1df6dad9 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -526,6 +526,29 @@ def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order, assert log_has_re(message, caplog) +def test_handle_protections(mocker, default_conf, fee): + default_conf['protections'] = [ + {"method": "CooldownPeriod", "stop_duration": 60}, + { + "method": "StoplossGuard", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 4, + "only_per_pair": False + } + ] + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade.protections._protection_handlers[1].global_stop = MagicMock( + return_value=(True, arrow.utcnow().shift(hours=1).datetime, "asdf")) + create_mock_trades(fee) + freqtrade.handle_protections('ETC/BTC') + send_msg_mock = freqtrade.rpc.send_msg + assert send_msg_mock.call_count == 2 + assert send_msg_mock.call_args_list[0][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER + assert send_msg_mock.call_args_list[1][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER_GLOBAL + + def test_create_trade_no_signal(default_conf, fee, mocker) -> None: default_conf['dry_run'] = True From 3ce05c0d548119fd5618ba2700313a69ec10f3c5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Sep 2021 20:08:48 +0200 Subject: [PATCH 122/239] Add "sane" defaults to protection triggers --- docs/telegram-usage.md | 1 + freqtrade/constants.py | 9 +++++++++ tests/rpc/test_rpc_telegram.py | 2 ++ 3 files changed, 12 insertions(+) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 1e6fa9dae..b9d01a236 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -105,6 +105,7 @@ Example configuration showing the different settings: `buy` notifications are sent when the order is placed, while `buy_fill` notifications are sent when the order is filled on the exchange. `sell` notifications are sent when the order is placed, while `sell_fill` notifications are sent when the order is filled on the exchange. `*_fill` notifications are off by default and must be explicitly enabled. +`protection_trigger` notifications are sent when a protection triggers and `protection_trigger_global` notifications trigger when global protections are triggered. `balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 9ca43d459..4997108bc 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -284,6 +284,15 @@ CONF_SCHEMA = { 'enum': TELEGRAM_SETTING_OPTIONS, 'default': 'off' }, + 'protection_trigger': { + 'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS, + 'default': 'off' + }, + 'protection_trigger_global': { + 'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS, + }, } }, 'reload': {'type': 'boolean'}, diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 8c285a76e..7dde7b803 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1315,6 +1315,8 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None: def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> None: + default_conf['telegram']['notification_settings']['protection_trigger'] = 'on' + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) time_machine.move_to("2021-09-01 05:00:00 +00:00") lock = PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=6).datetime, 'randreason') From abddb0db66b431abf5902a8fa74dbf5f2cf16e56 Mon Sep 17 00:00:00 2001 From: Bernhard Millauer Date: Tue, 21 Sep 2021 10:13:19 +0200 Subject: [PATCH 123/239] Fix header indention Co-authored-by: Matthias --- docs/docker_quickstart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index f4f8c366d..33b1c7ea1 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -151,7 +151,7 @@ You can then run `docker-compose build` to build the docker image, and run it us ### Troubleshooting -### Docker on Windows +#### Docker on Windows * Error: `"Timestamp for this request is outside of the recvWindow."` * The market api requests require a synchronized clock but the time in the docker container shifts a bit over time into the past. From 6fc770d97dd8d4380be3fb94e8f3dcedfa981cd7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 21 Sep 2021 15:12:35 +0200 Subject: [PATCH 124/239] Add warning about running with docker on windows --- docs/docker_quickstart.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 33b1c7ea1..2f350d207 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -163,6 +163,10 @@ You can then run `docker-compose build` to build the docker image, and run it us start "" "C:\Program Files\Docker\Docker\Docker Desktop.exe" ``` +!!! Warning + Due to the above, we do not recommend the usage of docker on windows for production setups, but only for experimentation, datadownload and backtesting. + Best use a linux-VPS for running freqtrade reliably. + ## Plotting with docker-compose Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your docker-compose.yml file. From 277828bf0ebac1c370b02b4dfb6ca15628d63b1e Mon Sep 17 00:00:00 2001 From: matt ferrante Date: Tue, 21 Sep 2021 07:56:16 -0600 Subject: [PATCH 125/239] parameterize some tests --- tests/test_freqtradebot.py | 219 ++++++++++++++----------------------- 1 file changed, 84 insertions(+), 135 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 72d1f6150..d96ef71d6 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3545,8 +3545,34 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f caplog) -def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fee, mocker): - trades_for_order[0]['fee']['currency'] = 'ETH' +@pytest.mark.parametrize( + 'fee_cost, fee_currency, fee_reduction_amount, use_ticker_rate, expected_log', + [ + (None, 'ETH', 0, True, None), + (0.004, None, 0, True, None), + (0.00094518, "BNB", 0, True, None), + ( + 0.004, + "LTC", + 0.004, + False, + ( + 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).' + ) + ), + (0.008, None, 0, True, None), + ] +) +def test_get_real_amount( + default_conf, trades_for_order, buy_order_fee, fee, mocker, caplog, + fee_cost, fee_currency, fee_reduction_amount, use_ticker_rate, expected_log +): + + buy_order = deepcopy(buy_order_fee) + buy_order['fee'] = {'cost': fee_cost, 'currency': fee_currency} + trades_for_order[0]['fee']['cost'] = fee_cost + trades_for_order[0]['fee']['currency'] = fee_currency mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) @@ -3561,19 +3587,58 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fe ) freqtrade = get_patched_freqtradebot(mocker, default_conf) + if not use_ticker_rate: + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', side_effect=ExchangeError) + # Amount does not change - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount + assert freqtrade.get_real_amount(trade, buy_order) == amount - fee_reduction_amount + + if expected_log: + assert log_has(expected_log, caplog) -def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_order_fee, - fee, mocker): +@pytest.mark.parametrize( + 'stake_currency, fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log', + [ + ( + "BTC", + None, + None, + 0.001, + 0.001, + ( + 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).' + ) + ), + ( + "ETH", + 0.02, + 'BNB', + 0.0005, + 0.001518575, + ( + 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).' + ) + ), + ] +) +def test_get_real_amount_multi( + default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker, markets, + stake_currency, fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log, +): - limit_buy_order = deepcopy(buy_order_fee) - limit_buy_order['fee'] = {'cost': 0.004, 'currency': None} - trades_for_order[0]['fee']['currency'] = None + trades_for_order = deepcopy(trades_for_order2) + if fee_cost: + trades_for_order[0]['fee']['cost'] = fee_cost + if fee_currency: + trades_for_order[0]['fee']['currency'] = fee_currency mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - amount = sum(x['amount'] for x in trades_for_order) + amount = float(sum(x['amount'] for x in trades_for_order)) + default_conf['stake_currency'] = stake_currency + trade = Trade( pair='LTC/ETH', amount=amount, @@ -3583,124 +3648,29 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_ open_rate=0.245441, open_order_id="123456" ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - # Amount does not change - assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - - -def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, fee, mocker): - trades_for_order[0]['fee']['currency'] = 'BNB' - trades_for_order[0]['fee']['cost'] = 0.00094518 - - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - amount = sum(x['amount'] for x in trades_for_order) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" - ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - - # Amount does not change - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - - -def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker): - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order2) - amount = float(sum(x['amount'] for x in trades_for_order2)) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" - ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - - # Amount is reduced by "fee" - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) - assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).', - caplog) - - assert trade.fee_open == 0.001 - assert trade.fee_close == 0.001 - assert trade.fee_open_cost is not None - assert trade.fee_open_currency is not None - assert trade.fee_close_cost is None - assert trade.fee_close_currency is None - - -def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, caplog, fee, - mocker, markets): - # Different fee currency on both trades - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order3) - amount = float(sum(x['amount'] for x in trades_for_order3)) - default_conf['stake_currency'] = 'ETH' - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" - ) # Fake markets entry to enable fee parsing markets['BNB/ETH'] = markets['ETH/BTC'] freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - return_value={'ask': 0.19, 'last': 0.2}) + mocker.patch( + 'freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': 0.19, 'last': 0.2} + ) # Amount is reduced by "fee" - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.0005) - assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).', - caplog) - # Overall fee is average of both trade's fee - assert trade.fee_open == 0.001518575 + expected_amount = amount - (amount * fee_reduction_amount) + assert freqtrade.get_real_amount(trade, buy_order_fee) == expected_amount + assert log_has(expected_log, caplog) + + assert trade.fee_open == expected_fee + assert trade.fee_close == expected_fee assert trade.fee_open_cost is not None assert trade.fee_open_currency is not None assert trade.fee_close_cost is None assert trade.fee_close_currency is None -def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, fee, - caplog, mocker): - limit_buy_order = deepcopy(buy_order_fee) - limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'} - - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', - return_value=[trades_for_order]) - amount = float(sum(x['amount'] for x in trades_for_order)) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" - ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - # Ticker rate cannot be found for this to work. - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', side_effect=ExchangeError) - - # Amount is reduced by "fee" - assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004 - assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).', - caplog) - - def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, fee, mocker): limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004} @@ -3768,27 +3738,6 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b abs_tol=MATH_CLOSE_PREC,) -def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, fee, mocker): - # Remove "Currency" from fee dict - trades_for_order[0]['fee'] = {'cost': 0.008} - - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - amount = sum(x['amount'] for x in trades_for_order) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - open_rate=0.245441, - fee_open=fee.return_value, - fee_close=fee.return_value, - - open_order_id="123456" - ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - # Amount does not change - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - - def test_get_real_amount_open_trade(default_conf, fee, mocker): amount = 12345 trade = Trade( From 707d0ef795868961fe652046f12a190047809de3 Mon Sep 17 00:00:00 2001 From: matt ferrante Date: Tue, 21 Sep 2021 12:16:10 -0600 Subject: [PATCH 126/239] remove trades_for_order3 --- tests/conftest.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5e08e7097..7354c0b2c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1685,14 +1685,6 @@ def trades_for_order2(): 'fee': {'cost': 0.004, 'currency': 'LTC'}}] -@pytest.fixture(scope="function") -def trades_for_order3(trades_for_order2): - # Different fee currencies for each trade - trades_for_order = deepcopy(trades_for_order2) - trades_for_order[0]['fee'] = {'cost': 0.02, 'currency': 'BNB'} - return trades_for_order - - @pytest.fixture def buy_order_fee(): return { From b0de4d333e86bfe23f065ab365e2ab18bb7ca41d Mon Sep 17 00:00:00 2001 From: Peter Willemsen Date: Tue, 21 Sep 2021 23:20:40 +0200 Subject: [PATCH 127/239] fixed webhook error --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1cb8988ff..95f2280ac 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1217,7 +1217,7 @@ class FreqtradeBot(LoggingMixin): 'exchange': trade.exchange.capitalize(), 'pair': trade.pair, 'gain': gain, - 'limit': profit_rate, + 'limit': profit_rate or 0, 'order_type': order_type, 'amount': trade.amount, 'open_rate': trade.open_rate, @@ -1226,7 +1226,7 @@ class FreqtradeBot(LoggingMixin): 'profit_ratio': profit_ratio, 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, - 'close_date': trade.close_date, + 'close_date': trade.close_date or datetime.utcnow(), 'stake_currency': self.config['stake_currency'], 'fiat_currency': self.config.get('fiat_display_currency', None), 'reason': reason, From 553c868d7f2a8d4e7edafaccdf814cc7249e0ac9 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Tue, 21 Sep 2021 16:40:24 -0600 Subject: [PATCH 128/239] combined test_order_book_depth_of_market and test_order_book_depth_of_market_high_delta --- tests/test_freqtradebot.py | 122 +++++++++++++------------------------ 1 file changed, 42 insertions(+), 80 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index d96ef71d6..0a2f73263 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3546,24 +3546,19 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f @pytest.mark.parametrize( - 'fee_cost, fee_currency, fee_reduction_amount, use_ticker_rate, expected_log', - [ + 'fee_cost, fee_currency, fee_reduction_amount, use_ticker_rate, expected_log', [ (None, 'ETH', 0, True, None), (0.004, None, 0, True, None), - (0.00094518, "BNB", 0, True, None), - ( - 0.004, - "LTC", - 0.004, - False, - ( - 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).' - ) - ), + (0.00094518, "BNB", 0, True, ( + 'Fee for Trade Trade(id=None, pair=LTC/ETH, amount=8.00000000, open_rate=0.24544100,' + ' open_since=closed) [buy]: 0.00094518 BNB - rate: None' + )), + (0.004, "LTC", 0.004, False, ( + 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).' + )), (0.008, None, 0, True, None), - ] -) + ]) def test_get_real_amount( default_conf, trades_for_order, buy_order_fee, fee, mocker, caplog, fee_cost, fee_currency, fee_reduction_amount, use_ticker_rate, expected_log @@ -3590,6 +3585,7 @@ def test_get_real_amount( if not use_ticker_rate: mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', side_effect=ExchangeError) + caplog.clear() # Amount does not change assert freqtrade.get_real_amount(trade, buy_order) == amount - fee_reduction_amount @@ -3598,32 +3594,16 @@ def test_get_real_amount( @pytest.mark.parametrize( - 'stake_currency, fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log', - [ - ( - "BTC", - None, - None, - 0.001, - 0.001, - ( - 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).' - ) - ), - ( - "ETH", - 0.02, - 'BNB', - 0.0005, - 0.001518575, - ( - 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).' - ) - ), - ] -) + 'stake_currency, fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log', [ + ("BTC", None, None, 0.001, 0.001, ( + 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).' + )), + ("ETH", 0.02, 'BNB', 0.0005, 0.001518575, ( + 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).' + )), + ]) def test_get_real_amount_multi( default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker, markets, stake_currency, fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log, @@ -3653,10 +3633,8 @@ def test_get_real_amount_multi( markets['BNB/ETH'] = markets['ETH/BTC'] freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - mocker.patch( - 'freqtrade.exchange.Exchange.fetch_ticker', - return_value={'ask': 0.19, 'last': 0.2} - ) + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': 0.19, 'last': 0.2}) # Amount is reduced by "fee" expected_amount = amount - (amount * fee_reduction_amount) @@ -3788,10 +3766,14 @@ def test_apply_fee_conditional(default_conf, fee, caplog, mocker, assert walletmock.call_count == 1 +@pytest.mark.parametrize("delta, is_high_delta", [ + (0.1, False), + (100, True), +]) def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open, limit_buy_order, - fee, mocker, order_book_l2): + fee, mocker, order_book_l2, delta, is_high_delta): default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True - default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 + default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = delta patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) @@ -3809,42 +3791,22 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open, freqtrade.enter_positions() trade = Trade.query.first() - assert trade is not None - assert trade.stake_amount == 0.001 - assert trade.is_open - assert trade.open_date is not None - assert trade.exchange == 'binance' + if is_high_delta: + assert trade is None + else: + assert trade is not None + assert trade.stake_amount == 0.001 + assert trade.is_open + assert trade.open_date is not None + assert trade.exchange == 'binance' - assert len(Trade.query.all()) == 1 + assert len(Trade.query.all()) == 1 - # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) - assert trade.open_rate == 0.00001099 - assert whitelist == default_conf['exchange']['pair_whitelist'] - - -def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_order, - fee, mocker, order_book_l2): - default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True - # delta is 100 which is impossible to reach. hence check_depth_of_market will return false - default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100 - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, - create_order=MagicMock(return_value={'id': limit_buy_order['id']}), - get_fee=fee, - ) - # Save state of current whitelist - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - freqtrade.enter_positions() - - trade = Trade.query.first() - assert trade is None + assert trade.open_rate == 0.00001099 + assert whitelist == default_conf['exchange']['pair_whitelist'] @pytest.mark.parametrize('exception_thrown,ask,last,order_book_top,order_book', [ From f768bdea503780b6c220afe36691fbbd09752689 Mon Sep 17 00:00:00 2001 From: matt ferrante Date: Wed, 22 Sep 2021 10:32:30 -0600 Subject: [PATCH 129/239] cleanup based on feedback --- tests/test_freqtradebot.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 0a2f73263..eaaadbcb1 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3594,19 +3594,13 @@ def test_get_real_amount( @pytest.mark.parametrize( - 'stake_currency, fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log', [ - ("BTC", None, None, 0.001, 0.001, ( - 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).' - )), - ("ETH", 0.02, 'BNB', 0.0005, 0.001518575, ( - 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).' - )), + 'stake_currency, fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount', [ + (None, None, None, 0.001, 0.001, 7.992), + ("ETH", 0.02, 'BNB', 0.0005, 0.001518575, 7.996), ]) def test_get_real_amount_multi( default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker, markets, - stake_currency, fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log, + stake_currency, fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount, ): trades_for_order = deepcopy(trades_for_order2) @@ -3617,7 +3611,8 @@ def test_get_real_amount_multi( mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = float(sum(x['amount'] for x in trades_for_order)) - default_conf['stake_currency'] = stake_currency + if stake_currency: + default_conf['stake_currency'] = stake_currency trade = Trade( pair='LTC/ETH', @@ -3639,7 +3634,13 @@ def test_get_real_amount_multi( # Amount is reduced by "fee" expected_amount = amount - (amount * fee_reduction_amount) assert freqtrade.get_real_amount(trade, buy_order_fee) == expected_amount - assert log_has(expected_log, caplog) + assert log_has( + ( + 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + f'open_rate=0.24544100, open_since=closed) (from 8.0 to {expected_log_amount}).' + ), + caplog + ) assert trade.fee_open == expected_fee assert trade.fee_close == expected_fee From 8cfb6ddd518bc4edf910c9f07afdc1ee85d103aa Mon Sep 17 00:00:00 2001 From: matt ferrante Date: Wed, 22 Sep 2021 10:48:13 -0600 Subject: [PATCH 130/239] fix long line --- tests/test_freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index eaaadbcb1..b987d54d8 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3594,7 +3594,7 @@ def test_get_real_amount( @pytest.mark.parametrize( - 'stake_currency, fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount', [ + 'stake_currency,fee_cost,fee_currency,fee_reduction_amount,expected_fee,expected_log_amount', [ (None, None, None, 0.001, 0.001, 7.992), ("ETH", 0.02, 'BNB', 0.0005, 0.001518575, 7.996), ]) From 30cc69c880cf736225f6246331527ab202398588 Mon Sep 17 00:00:00 2001 From: matt ferrante Date: Wed, 22 Sep 2021 11:28:42 -0600 Subject: [PATCH 131/239] set all to eth for multi test --- tests/test_freqtradebot.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index b987d54d8..8e036e80a 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3594,13 +3594,13 @@ def test_get_real_amount( @pytest.mark.parametrize( - 'stake_currency,fee_cost,fee_currency,fee_reduction_amount,expected_fee,expected_log_amount', [ - (None, None, None, 0.001, 0.001, 7.992), - ("ETH", 0.02, 'BNB', 0.0005, 0.001518575, 7.996), + 'fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount', [ + (None, None, 0.001, 0.001, 7.992), + (0.02, 'BNB', 0.0005, 0.001518575, 7.996), ]) def test_get_real_amount_multi( default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker, markets, - stake_currency, fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount, + fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount, ): trades_for_order = deepcopy(trades_for_order2) @@ -3611,8 +3611,7 @@ def test_get_real_amount_multi( mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = float(sum(x['amount'] for x in trades_for_order)) - if stake_currency: - default_conf['stake_currency'] = stake_currency + default_conf['stake_currency'] = "ETH" trade = Trade( pair='LTC/ETH', From 2bf49445b7db8a15681f8fb690d3db6162f0e8c1 Mon Sep 17 00:00:00 2001 From: matt ferrante Date: Wed, 22 Sep 2021 16:11:27 -0600 Subject: [PATCH 132/239] add parameterized names --- tests/test_freqtradebot.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 8e036e80a..9c9271810 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3547,16 +3547,21 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f @pytest.mark.parametrize( 'fee_cost, fee_currency, fee_reduction_amount, use_ticker_rate, expected_log', [ + # basic, amount does not change (None, 'ETH', 0, True, None), + # no currency in fee (0.004, None, 0, True, None), + # BNB no rate (0.00094518, "BNB", 0, True, ( 'Fee for Trade Trade(id=None, pair=LTC/ETH, amount=8.00000000, open_rate=0.24544100,' ' open_since=closed) [buy]: 0.00094518 BNB - rate: None' )), + # from order (0.004, "LTC", 0.004, False, ( 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).' )), + # invalid, no currency in from fee dict (0.008, None, 0, True, None), ]) def test_get_real_amount( @@ -3586,7 +3591,6 @@ def test_get_real_amount( mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', side_effect=ExchangeError) caplog.clear() - # Amount does not change assert freqtrade.get_real_amount(trade, buy_order) == amount - fee_reduction_amount if expected_log: @@ -3595,7 +3599,9 @@ def test_get_real_amount( @pytest.mark.parametrize( 'fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount', [ + # basic, amount is reduced by fee (None, None, 0.001, 0.001, 7.992), + # different fee currency on both trades, fee is average of both trade's fee (0.02, 'BNB', 0.0005, 0.001518575, 7.996), ]) def test_get_real_amount_multi( From d7903f012f7af5f3c19df527780456bab0ca986f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Sep 2021 07:25:11 +0200 Subject: [PATCH 133/239] Move PerformanceWarning to advanced section rewrite to use strategy parameters instead of plain range --- docs/strategy-advanced.md | 30 ++++++++++++++++++++++++++++++ docs/strategy-customization.md | 27 --------------------------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 4409af6ea..d7d0dd04a 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -695,3 +695,33 @@ The variable 'content', will contain the strategy file in a BASE64 encoded form. ``` Please ensure that 'NameOfStrategy' is identical to the strategy name! + +## Performance warning + +When executing a strategy, one can sometimes be greeted by the following in the logs + +> PerformanceWarning: DataFrame is highly fragmented. + +This is a warning from [`pandas`](https://github.com/pandas-dev/pandas) and as the warning continues to say: +use `pd.concat(axis=1)`. +This can have slight performance implications, which are usually only visible during hyperopt (when optimizing an indicator). + +For example: + +```python +for val in self.buy_ema_short.range: + dataframe[f'ema_short_{val}'] = ta.EMA(dataframe, timeperiod=val) +``` + +should be rewritten to + +```python +frames = [dataframe] +for val in self.buy_ema_short.range: + frames.append({ + f'ema_short_{val}': ta.EMA(dataframe, timeperiod=val) + }) + +# Append columns to existing dataframe +merged_frame = pd.concat(frames, axis=1) +``` diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 194517b2b..6d6eff2aa 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -793,33 +793,6 @@ The following lists some common patterns which should be avoided to prevent frus - don't use `dataframe['volume'].mean()`. This uses the full DataFrame for backtesting, including data from the future. Use `dataframe['volume'].rolling().mean()` instead - don't use `.resample('1h')`. This uses the left border of the interval, so moves data from an hour to the start of the hour. Use `.resample('1h', label='right')` instead. -### Performance warning - -When executing a strategy, one can sometimes be greeted by the following in the logs - -> PerformanceWarning: DataFrame is highly fragmented. - -This is a warning from [`pandas`](https://github.com/pandas-dev/pandas) and as the warning continues to say: - use `pd.concat(axis=1)`. For example - -```python -for i in range(100): - dataframe[i] = ta.indicator(dataframe, param=i) -``` - -should be rewritten to - -```python -frames = [dataframe] -for i in range(100): - frames.append({ - str(i): ta.indicator(dataframe, param=i) - }) - -# Append columns to existing dataframe -merged_frame = pd.concat(frames, axis=1) -``` - ## Further strategy ideas To get additional Ideas for strategies, head over to our [strategy repository](https://github.com/freqtrade/freqtrade-strategies). Feel free to use them as they are - but results will depend on the current market situation, pairs used etc. - therefore please backtest the strategy for your exchange/desired pairs first, evaluate carefully, use at your own risk. From 692e91a26dfa2ec1c746923d481421cc87a15b8c Mon Sep 17 00:00:00 2001 From: Peter Willemsen Date: Thu, 23 Sep 2021 10:28:15 +0200 Subject: [PATCH 134/239] changed close date from datetime.utcnow() to datetime.now(timezone.utc) --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 95f2280ac..9cfd219a8 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1226,7 +1226,7 @@ class FreqtradeBot(LoggingMixin): 'profit_ratio': profit_ratio, 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, - 'close_date': trade.close_date or datetime.utcnow(), + 'close_date': trade.close_date or datetime.now(timezone.utc), 'stake_currency': self.config['stake_currency'], 'fiat_currency': self.config.get('fiat_display_currency', None), 'reason': reason, From 422d5601890f9749ab7f69a903d2de3f68ea9257 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 05:05:55 +0000 Subject: [PATCH 135/239] Bump types-requests from 2.25.6 to 2.25.8 Bumps [types-requests](https://github.com/python/typeshed) from 2.25.6 to 2.25.8. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4859e1cc6..d8d8ce916 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,5 +23,5 @@ nbconvert==6.1.0 # mypy types types-cachetools==4.2.0 types-filelock==0.1.5 -types-requests==2.25.6 +types-requests==2.25.8 types-tabulate==0.8.2 From e85dc6326302e83da897b06941ca725f6d5f242c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 05:06:01 +0000 Subject: [PATCH 136/239] Bump mkdocs-material from 7.2.6 to 7.3.0 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 7.2.6 to 7.3.0. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/docs/changelog.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/7.2.6...7.3.0) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 9927740c2..9b7c12a43 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.2 -mkdocs-material==7.2.6 +mkdocs-material==7.3.0 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 From 766ef90b56cf190b8beeba1a249d9b53792fbba2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 05:06:16 +0000 Subject: [PATCH 137/239] Bump sqlalchemy from 1.4.23 to 1.4.25 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.23 to 1.4.25. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/master/CHANGES) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aa729dd9f..74ce391ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ ccxt==1.56.30 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.8 aiohttp==3.7.4.post0 -SQLAlchemy==1.4.23 +SQLAlchemy==1.4.25 python-telegram-bot==13.7 arrow==1.1.1 cachetools==4.2.2 From 90d5af9a35211a51756603cfb3ffcb3cf67edc17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 05:06:20 +0000 Subject: [PATCH 138/239] Bump urllib3 from 1.26.6 to 1.26.7 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.6 to 1.26.7. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/1.26.7/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.6...1.26.7) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aa729dd9f..2f6a4a104 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ python-telegram-bot==13.7 arrow==1.1.1 cachetools==4.2.2 requests==2.26.0 -urllib3==1.26.6 +urllib3==1.26.7 wrapt==1.12.1 jsonschema==3.2.0 TA-Lib==0.4.21 From 0353f070f9714655e1ccb06448ffa03dbe3f5fb0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 05:06:24 +0000 Subject: [PATCH 139/239] Bump progressbar2 from 3.53.2 to 3.53.3 Bumps [progressbar2](https://github.com/WoLpH/python-progressbar) from 3.53.2 to 3.53.3. - [Release notes](https://github.com/WoLpH/python-progressbar/releases) - [Changelog](https://github.com/WoLpH/python-progressbar/blob/develop/CHANGES.rst) - [Commits](https://github.com/WoLpH/python-progressbar/compare/v3.53.2...v3.53.3) --- updated-dependencies: - dependency-name: progressbar2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 7dc55a9fc..9feec80f1 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -8,4 +8,4 @@ scikit-optimize==0.8.1 filelock==3.0.12 joblib==1.0.1 psutil==5.8.0 -progressbar2==3.53.2 +progressbar2==3.53.3 From 954c468191b3c130c62d7bedfde1ff138f427908 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 24 Sep 2021 07:12:38 +0200 Subject: [PATCH 140/239] Add pandas-ta to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index aa729dd9f..f0da0cdb9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ numpy==1.21.2 pandas==1.3.3 +pandas-ta==0.3.14b ccxt==1.56.30 # Pin cryptography for now due to rust build errors with piwheels From 72a1e27fc6cc0454dfb3035f4957820dd6c6423d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 07:31:20 +0000 Subject: [PATCH 141/239] Bump ccxt from 1.56.30 to 1.56.86 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.56.30 to 1.56.86. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.56.30...1.56.86) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3b004dfaa..ede0c74e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.21.2 pandas==1.3.3 pandas-ta==0.3.14b -ccxt==1.56.30 +ccxt==1.56.86 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.8 aiohttp==3.7.4.post0 From b59906b117f300aefd828fa02c8b5c5658388df1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 24 Sep 2021 19:24:33 +0200 Subject: [PATCH 142/239] Update minimum for tradable_balance_ratio to 0.0 --- freqtrade/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 4997108bc..fca319a0f 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -110,7 +110,7 @@ CONF_SCHEMA = { }, 'tradable_balance_ratio': { 'type': 'number', - 'minimum': 0.1, + 'minimum': 0.0, 'maximum': 1, 'default': 0.99 }, From 4c268847d42043d53fc59751ffd1717125d9e0bd Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 24 Sep 2021 19:32:22 +0200 Subject: [PATCH 143/239] Add pandas-ta to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 727c40c7c..cf381bdd3 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ setup( 'wrapt', 'jsonschema', 'TA-Lib', + 'pandas-ta', 'technical', 'tabulate', 'pycoingecko', From f4f204d849eb65eed74a1ac09e1ad52e5a641ee9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 24 Sep 2021 20:09:24 +0200 Subject: [PATCH 144/239] Update test to use cost dict --- tests/test_freqtradebot.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 9c9271810..b233a6267 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3546,33 +3546,32 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f @pytest.mark.parametrize( - 'fee_cost, fee_currency, fee_reduction_amount, use_ticker_rate, expected_log', [ + 'fee_par,fee_reduction_amount,use_ticker_rate,expected_log', [ # basic, amount does not change - (None, 'ETH', 0, True, None), + ({'cost': 0.008, 'currency': 'ETH'}, 0, False, None), # no currency in fee - (0.004, None, 0, True, None), + ({'cost': 0.004, 'currency': None}, 0, True, None), # BNB no rate - (0.00094518, "BNB", 0, True, ( + ({'cost': 0.00094518, 'currency': 'BNB'}, 0, True, ( 'Fee for Trade Trade(id=None, pair=LTC/ETH, amount=8.00000000, open_rate=0.24544100,' ' open_since=closed) [buy]: 0.00094518 BNB - rate: None' )), # from order - (0.004, "LTC", 0.004, False, ( + ({'cost': 0.004, 'currency': 'LTC'}, 0.004, False, ( 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).' )), # invalid, no currency in from fee dict - (0.008, None, 0, True, None), + ({'cost': 0.008, 'currency': None}, 0, True, None), ]) def test_get_real_amount( default_conf, trades_for_order, buy_order_fee, fee, mocker, caplog, - fee_cost, fee_currency, fee_reduction_amount, use_ticker_rate, expected_log + fee_par, fee_reduction_amount, use_ticker_rate, expected_log ): buy_order = deepcopy(buy_order_fee) - buy_order['fee'] = {'cost': fee_cost, 'currency': fee_currency} - trades_for_order[0]['fee']['cost'] = fee_cost - trades_for_order[0]['fee']['currency'] = fee_currency + buy_order['fee'] = fee_par + trades_for_order[0]['fee'] = fee_par mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) From 097da448e2d297ace7f2158efb4fac6164168028 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sat, 25 Sep 2021 15:48:42 +0100 Subject: [PATCH 145/239] Add CPU,RAM sysinfo support to the REST API to help with bot system monitoring --- freqtrade/rpc/api_server/api_v1.py | 4 ++++ freqtrade/rpc/rpc.py | 5 ++++- requirements.txt | 1 + scripts/rest_client.py | 6 ++++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 7e613f184..733fa7383 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -259,3 +259,7 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option 'pair_interval': pair_interval, } return result + +@router.get('/sysinfo', tags=['info']) +def sysinfo(rpc: RPC = Depends(get_rpc)): + return rpc._rpc_sysinfo() diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f6599b429..9b0d4b0f7 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -1,7 +1,7 @@ """ This module contains class to define a RPC communications """ -import logging +import logging, psutil from abc import abstractmethod from datetime import date, datetime, timedelta, timezone from math import isnan @@ -870,3 +870,6 @@ class RPC: 'subplots' not in self._freqtrade.strategy.plot_config): self._freqtrade.strategy.plot_config['subplots'] = {} return self._freqtrade.strategy.plot_config + + def _rpc_sysinfo(self) -> Dict[str, Any]: + return {"cpu_pct": psutil.cpu_percent(interval=1, percpu=True), "ram_pct": psutil.virtual_memory().percent} diff --git a/requirements.txt b/requirements.txt index d1d10dd1d..6a2cfec01 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,6 +36,7 @@ fastapi==0.68.1 uvicorn==0.15.0 pyjwt==2.1.0 aiofiles==0.7.0 +psutil==5.8.0 # Support for colorized terminal output colorama==0.4.4 diff --git a/scripts/rest_client.py b/scripts/rest_client.py index ece0a253e..52de3c534 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -334,6 +334,12 @@ class FtRestClient(): "timerange": timerange if timerange else '', }) + def sysinfo(self): + """Provides system information (CPU, RAM usage) + + :return: json object + """ + return self._get("sysinfo") def add_arguments(): parser = argparse.ArgumentParser() From 6319c104fe1c51e0343c08def9f4511cc537b377 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Sep 2021 15:07:48 +0200 Subject: [PATCH 146/239] Fix unreliable backtest-result when using webserver mode --- freqtrade/data/dataprovider.py | 2 + freqtrade/optimize/backtesting.py | 50 ++++++++++++------------ freqtrade/rpc/api_server/api_backtest.py | 21 +++++----- 3 files changed, 39 insertions(+), 34 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index cdee0f078..b197c159f 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -149,6 +149,8 @@ class DataProvider: Clear pair dataframe cache. """ self.__cached_pairs = {} + self.__cached_pairs_backtesting = {} + self.__slice_index = 0 # Exchange functions diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 79c861ee8..f406f89d7 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -85,18 +85,7 @@ class Backtesting: "configuration or as cli argument `--timeframe 5m`") self.timeframe = str(self.config.get('timeframe')) self.timeframe_min = timeframe_to_minutes(self.timeframe) - # Load detail timeframe if specified - self.timeframe_detail = str(self.config.get('timeframe_detail', '')) - if self.timeframe_detail: - self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail) - if self.timeframe_min <= self.timeframe_detail_min: - raise OperationalException( - "Detail timeframe must be smaller than strategy timeframe.") - - else: - self.timeframe_detail_min = 0 - self.detail_data: Dict[str, DataFrame] = {} - + self.init_backtest_detail() self.pairlists = PairListManager(self.exchange, self.config) if 'VolumePairList' in self.pairlists.name_list: raise OperationalException("VolumePairList not allowed for backtesting.") @@ -119,14 +108,6 @@ class Backtesting: else: self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0]) - Trade.use_db = False - Trade.reset_trades() - PairLocks.timeframe = self.config['timeframe'] - PairLocks.use_db = False - PairLocks.reset_locks() - - self.wallets = Wallets(self.config, self.exchange, log=False) - self.timerange = TimeRange.parse_timerange( None if self.config.get('timerange') is None else str(self.config.get('timerange'))) @@ -135,9 +116,7 @@ class Backtesting: # Add maximum startup candle count to configuration for informative pairs support self.config['startup_candle_count'] = self.required_startup self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe) - - self.progress = BTProgress() - self.abort = False + self.init_backtest() def __del__(self): self.cleanup() @@ -147,6 +126,28 @@ class Backtesting: PairLocks.use_db = True Trade.use_db = True + def init_backtest_detail(self): + # Load detail timeframe if specified + self.timeframe_detail = str(self.config.get('timeframe_detail', '')) + if self.timeframe_detail: + self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail) + if self.timeframe_min <= self.timeframe_detail_min: + raise OperationalException( + "Detail timeframe must be smaller than strategy timeframe.") + + else: + self.timeframe_detail_min = 0 + self.detail_data: Dict[str, DataFrame] = {} + + def init_backtest(self): + + self.prepare_backtest(False) + + self.wallets = Wallets(self.config, self.exchange, log=False) + + self.progress = BTProgress() + self.abort = False + def _set_strategy(self, strategy: IStrategy): """ Load strategy into backtesting @@ -226,7 +227,8 @@ class Backtesting: Trade.reset_trades() self.rejected_trades = 0 self.dataprovider.clear_cache() - self._load_protections(self.strategy) + if enable_protections: + self._load_protections(self.strategy) def check_abort(self): """ diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 4623c187e..32278686c 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -47,33 +47,34 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac not ApiServer._bt or lastconfig.get('timeframe') != strat.timeframe or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail') - or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0) or lastconfig.get('timerange') != btconfig['timerange'] ): from freqtrade.optimize.backtesting import Backtesting ApiServer._bt = Backtesting(btconfig) if ApiServer._bt.timeframe_detail: ApiServer._bt.load_bt_data_detail() - + else: + ApiServer._bt.config = btconfig + ApiServer._bt.init_backtest() # Only reload data if timeframe changed. if ( not ApiServer._bt_data or not ApiServer._bt_timerange - or lastconfig.get('stake_amount') != btconfig.get('stake_amount') - or lastconfig.get('enable_protections') != btconfig.get('enable_protections') - or lastconfig.get('protections') != btconfig.get('protections', []) or lastconfig.get('timeframe') != strat.timeframe ): - lastconfig['timerange'] = btconfig['timerange'] - lastconfig['protections'] = btconfig.get('protections', []) - lastconfig['enable_protections'] = btconfig.get('enable_protections') - lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') - lastconfig['timeframe'] = strat.timeframe ApiServer._bt_data, ApiServer._bt_timerange = ApiServer._bt.load_bt_data() + lastconfig['timerange'] = btconfig['timerange'] + lastconfig['timeframe'] = strat.timeframe + + lastconfig['protections'] = btconfig.get('protections', []) + lastconfig['enable_protections'] = btconfig.get('enable_protections') + lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') + ApiServer._bt.abort = False min_date, max_date = ApiServer._bt.backtest_one_strategy( strat, ApiServer._bt_data, ApiServer._bt_timerange) + ApiServer._bt.results = generate_backtest_stats( ApiServer._bt_data, ApiServer._bt.all_results, min_date=min_date, max_date=max_date) From 08b1f04ed5f9b7ce69827fb44acfc4c154aecc2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 03:01:18 +0000 Subject: [PATCH 147/239] Bump types-requests from 2.25.8 to 2.25.9 Bumps [types-requests](https://github.com/python/typeshed) from 2.25.8 to 2.25.9. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d8d8ce916..1d61369eb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,5 +23,5 @@ nbconvert==6.1.0 # mypy types types-cachetools==4.2.0 types-filelock==0.1.5 -types-requests==2.25.8 +types-requests==2.25.9 types-tabulate==0.8.2 From 905950230329688b81606c9fb78c3e5a631d8cbd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 03:01:27 +0000 Subject: [PATCH 148/239] Bump ccxt from 1.56.86 to 1.57.3 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.56.86 to 1.57.3. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.56.86...1.57.3) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d1d10dd1d..feeb4d942 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.21.2 pandas==1.3.3 pandas-ta==0.3.14b -ccxt==1.56.86 +ccxt==1.57.3 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.8 aiohttp==3.7.4.post0 From 78096c9eff08cc2aea47d44e7ae20751c0b3420a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 04:32:26 +0000 Subject: [PATCH 149/239] Bump nbconvert from 6.1.0 to 6.2.0 Bumps [nbconvert](https://github.com/jupyter/nbconvert) from 6.1.0 to 6.2.0. - [Release notes](https://github.com/jupyter/nbconvert/releases) - [Commits](https://github.com/jupyter/nbconvert/compare/6.1.0...6.2.0) --- updated-dependencies: - dependency-name: nbconvert dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1d61369eb..2f03255a0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,7 +18,7 @@ isort==5.9.3 time-machine==2.4.0 # Convert jupyter notebooks to markdown documents -nbconvert==6.1.0 +nbconvert==6.2.0 # mypy types types-cachetools==4.2.0 From 5b7a1f864257c62924760be45f2dc9c651c665a0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 27 Sep 2021 07:12:40 +0200 Subject: [PATCH 150/239] Validate config also in webserver mode --- freqtrade/rpc/api_server/api_backtest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 32278686c..7ce9f487f 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -4,6 +4,7 @@ from copy import deepcopy from fastapi import APIRouter, BackgroundTasks, Depends +from freqtrade.configuration.config_validation import validate_config_consistency from freqtrade.enums import BacktestState from freqtrade.exceptions import DependencyException from freqtrade.rpc.api_server.api_schemas import BacktestRequest, BacktestResponse @@ -42,6 +43,7 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac # Reload strategy lastconfig = ApiServer._bt_last_config strat = StrategyResolver.load_strategy(btconfig) + validate_config_consistency(btconfig) if ( not ApiServer._bt From 3fbf716f85b96a8d53f99523cd50ab703c034256 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 27 Sep 2021 17:51:01 +0200 Subject: [PATCH 151/239] Fix "sticking" timerange in webserver mode --- freqtrade/rpc/api_server/api_backtest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 7ce9f487f..edbc39772 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -63,12 +63,12 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac not ApiServer._bt_data or not ApiServer._bt_timerange or lastconfig.get('timeframe') != strat.timeframe + or lastconfig.get('timerange') != btconfig['timerange'] ): ApiServer._bt_data, ApiServer._bt_timerange = ApiServer._bt.load_bt_data() - lastconfig['timerange'] = btconfig['timerange'] - lastconfig['timeframe'] = strat.timeframe - + lastconfig['timerange'] = btconfig['timerange'] + lastconfig['timeframe'] = strat.timeframe lastconfig['protections'] = btconfig.get('protections', []) lastconfig['enable_protections'] = btconfig.get('enable_protections') lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') From 5726886b0620874be0d1c6c8f8b4737912e42786 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 27 Sep 2021 20:52:19 +0200 Subject: [PATCH 152/239] Reduce backtest-noise from "pandas slice" warning --- freqtrade/optimize/backtesting.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f406f89d7..8328d61d3 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -385,12 +385,12 @@ class Backtesting: detail_data = detail_data.loc[ (detail_data['date'] >= sell_candle_time) & (detail_data['date'] < sell_candle_end) - ] + ].copy() if len(detail_data) == 0: # Fall back to "regular" data if no detail data was found for this candle return self._get_sell_trade_entry_for_candle(trade, sell_row) - detail_data['buy'] = sell_row[BUY_IDX] - detail_data['sell'] = sell_row[SELL_IDX] + detail_data.loc[:, 'buy'] = sell_row[BUY_IDX] + detail_data.loc[:, 'sell'] = sell_row[SELL_IDX] headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high'] for det_row in detail_data[headers].values.tolist(): res = self._get_sell_trade_entry_for_candle(trade, det_row) From e025576d8cac9cb0227734efc18d4c978d01cc6f Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Wed, 29 Sep 2021 10:15:05 +0300 Subject: [PATCH 153/239] Introduce markets_static fixture serving an immutable list of markets. Adapt pairlist/markets tests to use this new fixture. This allows freely modifying markets in get_markets() without a need of updating pairlist/markets tests. --- tests/commands/test_commands.py | 9 ++++----- tests/conftest.py | 21 ++++++++++++++++++--- tests/exchange/test_exchange.py | 4 ++-- tests/plugins/test_pairlist.py | 4 ++-- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 135510b38..b236f6a10 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -208,11 +208,10 @@ def test_list_timeframes(mocker, capsys): assert re.search(r"^1d$", captured.out, re.MULTILINE) -def test_list_markets(mocker, markets, capsys): +def test_list_markets(mocker, markets_static, capsys): api_mock = MagicMock() - api_mock.markets = markets - patch_exchange(mocker, api_mock=api_mock, id='bittrex') + patch_exchange(mocker, api_mock=api_mock, id='bittrex', mock_markets=markets_static) # Test with no --config args = [ @@ -237,7 +236,7 @@ def test_list_markets(mocker, markets, capsys): "TKN/BTC, XLTCUSDT, XRP/BTC.\n" in captured.out) - patch_exchange(mocker, api_mock=api_mock, id="binance") + patch_exchange(mocker, api_mock=api_mock, id="binance", mock_markets=markets_static) # Test with --exchange args = [ "list-markets", @@ -250,7 +249,7 @@ def test_list_markets(mocker, markets, capsys): assert re.match("\nExchange Binance has 10 active markets:\n", captured.out) - patch_exchange(mocker, api_mock=api_mock, id="bittrex") + patch_exchange(mocker, api_mock=api_mock, id="bittrex", mock_markets=markets_static) # Test with --all: all markets args = [ "list-markets", "--all", diff --git a/tests/conftest.py b/tests/conftest.py index 7354c0b2c..c908c0cb0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -90,8 +90,10 @@ def patch_exchange(mocker, api_mock=None, id='binance', mock_markets=True) -> No mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value=id.title())) mocker.patch('freqtrade.exchange.Exchange.precisionMode', PropertyMock(return_value=2)) if mock_markets: + if isinstance(mock_markets, bool): + mock_markets = get_markets() mocker.patch('freqtrade.exchange.Exchange.markets', - PropertyMock(return_value=get_markets())) + PropertyMock(return_value=mock_markets)) if api_mock: mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) @@ -376,6 +378,8 @@ def markets(): def get_markets(): + # See get_markets_static() for immutable markets and do not modify them unless absolutely + # necessary! return { 'ETH/BTC': { 'id': 'ethbtc', @@ -675,11 +679,22 @@ def get_markets(): @pytest.fixture -def shitcoinmarkets(markets): +def markets_static(): + # These markets are used in some tests that would need adaptation should anything change in + # market list. Do not modify this list without a good reason! Do not modify market parameters + # of listed pairs in get_markets() without a good reason either! + static_markets = ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', + 'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XLTCUSDT', 'XRP/BTC'] + all_markets = get_markets() + return {m: all_markets[m] for m in static_markets} + + +@pytest.fixture +def shitcoinmarkets(markets_static): """ Fixture with shitcoin markets - used to test filters in pairlists """ - shitmarkets = deepcopy(markets) + shitmarkets = deepcopy(markets_static) shitmarkets.update({ 'HOT/BTC': { 'id': 'HOTBTC', diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 97bc33429..79b4a3ff5 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2735,7 +2735,7 @@ def test_get_valid_pair_combination(default_conf, mocker, markets): (['LTC'], ['NONEXISTENT'], False, False, []), ]) -def test_get_markets(default_conf, mocker, markets, +def test_get_markets(default_conf, mocker, markets_static, base_currencies, quote_currencies, pairs_only, active_only, expected_keys): mocker.patch.multiple('freqtrade.exchange.Exchange', @@ -2743,7 +2743,7 @@ def test_get_markets(default_conf, mocker, markets, _load_async_markets=MagicMock(), validate_pairs=MagicMock(), validate_timeframes=MagicMock(), - markets=PropertyMock(return_value=markets)) + markets=PropertyMock(return_value=markets_static)) ex = Exchange(default_conf) pairs = ex.get_markets(base_currencies, quote_currencies, pairs_only, active_only) assert sorted(pairs.keys()) == sorted(expected_keys) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 1ce8d172c..cf918e2a0 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -131,9 +131,9 @@ def test_load_pairlist_noexist(mocker, markets, default_conf): default_conf, {}, 1) -def test_load_pairlist_verify_multi(mocker, markets, default_conf): +def test_load_pairlist_verify_multi(mocker, markets_static, default_conf): freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) + mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets_static)) plm = PairListManager(freqtrade.exchange, default_conf) # Call different versions one after the other, should always consider what was passed in # and have no side-effects (therefore the same check multiple times) From 656526c007dfa8fd80036966d601e8b873d1ccd5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Sep 2021 16:50:05 +0200 Subject: [PATCH 154/239] Add trades-to-ohlcv command to simplify adding new timeframes --- freqtrade/commands/__init__.py | 4 +-- freqtrade/commands/arguments.py | 14 +++++++++- freqtrade/commands/data_commands.py | 41 +++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index a6f14cff7..858c99acd 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -8,8 +8,8 @@ Note: Be careful with file-scoped imports in these subfiles. """ from freqtrade.commands.arguments import Arguments from freqtrade.commands.build_config_commands import start_new_config -from freqtrade.commands.data_commands import (start_convert_data, start_download_data, - start_list_data) +from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades, + start_download_data, start_list_data) from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui, start_new_strategy) from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index d424f3ce7..48dc48cf1 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -58,6 +58,8 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] +ARGS_CONVERT_TRADES = ["pairs", "timeframes", "dataformat_ohlcv", "dataformat_trades"] + ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange", @@ -169,7 +171,8 @@ class Arguments: self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot') self._build_args(optionlist=['version'], parser=self.parser) - from freqtrade.commands import (start_backtesting, start_convert_data, start_create_userdir, + from freqtrade.commands import (start_backtesting, start_convert_data, start_convert_trades, + start_create_userdir, start_download_data, start_edge, start_hyperopt, start_hyperopt_list, start_hyperopt_show, start_install_ui, start_list_data, start_list_exchanges, start_list_markets, @@ -236,6 +239,15 @@ class Arguments: convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False)) self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd) + # Add trades-to-ohlcv subcommand + convert_trade_data_cmd = subparsers.add_parser( + 'trades-to-ohlcv', + help='Convert trade data to OHLCV data.', + parents=[_common_parser], + ) + convert_trade_data_cmd.set_defaults(func=start_convert_trades) + self._build_args(optionlist=ARGS_CONVERT_TRADES, parser=convert_trade_data_cmd) + # Add list-data subcommand list_data_cmd = subparsers.add_parser( 'list-data', diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 141e85f14..7ef1ae5c7 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -89,6 +89,47 @@ def start_download_data(args: Dict[str, Any]) -> None: f"on exchange {exchange.name}.") +def start_convert_trades(args: Dict[str, Any]) -> None: + + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + timerange = TimeRange() + if 'days' in config: + time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d") + timerange = TimeRange.parse_timerange(f'{time_since}-') + + if 'timerange' in config: + timerange = timerange.parse_timerange(config['timerange']) + + # Remove stake-currency to skip checks which are not relevant for datadownload + config['stake_currency'] = '' + + if 'pairs' not in config: + raise OperationalException( + "Downloading data requires a list of pairs. " + "Please check the documentation on how to configure this.") + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + # Manual validations of relevant settings + if not config['exchange'].get('skip_pair_validation', False): + exchange.validate_pairs(config['pairs']) + expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets)) + + logger.info(f"About to Convert pairs: {expanded_pairs}, " + f"intervals: {config['timeframes']} to {config['datadir']}") + + for timeframe in config['timeframes']: + exchange.validate_timeframes(timeframe) + # Convert downloaded trade data to different timeframes + convert_trades_to_ohlcv( + pairs=expanded_pairs, timeframes=config['timeframes'], + datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), + data_format_ohlcv=config['dataformat_ohlcv'], + data_format_trades=config['dataformat_trades'], + ) + + def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: """ Convert data from one format to another From fc511aac4486dc99568e88edd01635453d0c1a59 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Sep 2021 19:21:54 +0200 Subject: [PATCH 155/239] don't use %default when no default is defined --- freqtrade/commands/cli_options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index e3c7fe464..d350a9426 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -381,12 +381,12 @@ AVAILABLE_CLI_OPTIONS = { ), "dataformat_ohlcv": Arg( '--data-format-ohlcv', - help='Storage format for downloaded candle (OHLCV) data. (default: `%(default)s`).', + help='Storage format for downloaded candle (OHLCV) data. (default: `json`).', choices=constants.AVAILABLE_DATAHANDLERS, ), "dataformat_trades": Arg( '--data-format-trades', - help='Storage format for downloaded trades data. (default: `%(default)s`).', + help='Storage format for downloaded trades data. (default: `jsongz`).', choices=constants.AVAILABLE_DATAHANDLERS, ), "exchange": Arg( From 248c61bb26399d3049f7ac2f116d6cf0e49d8665 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Sep 2021 19:39:29 +0200 Subject: [PATCH 156/239] Add test for trades-to-ohlcv --- freqtrade/commands/arguments.py | 4 ++-- freqtrade/commands/data_commands.py | 6 ------ tests/commands/test_commands.py | 28 ++++++++++++++++++++++------ 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 48dc48cf1..2fadf047e 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -58,7 +58,7 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] -ARGS_CONVERT_TRADES = ["pairs", "timeframes", "dataformat_ohlcv", "dataformat_trades"] +ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"] ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"] @@ -93,7 +93,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-data", "hyperopt-list", "hyperopt-show", - "plot-dataframe", "plot-profit", "show-trades"] + "plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 7ef1ae5c7..ee05e6c69 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -94,12 +94,6 @@ def start_convert_trades(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) timerange = TimeRange() - if 'days' in config: - time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d") - timerange = TimeRange.parse_timerange(f'{time_since}-') - - if 'timerange' in config: - timerange = timerange.parse_timerange(config['timerange']) # Remove stake-currency to skip checks which are not relevant for datadownload config['stake_currency'] = '' diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index b236f6a10..8889617ba 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -8,12 +8,12 @@ from zipfile import ZipFile import arrow import pytest -from freqtrade.commands import (start_convert_data, start_create_userdir, start_download_data, - start_hyperopt_list, start_hyperopt_show, start_install_ui, - start_list_data, start_list_exchanges, start_list_markets, - start_list_strategies, start_list_timeframes, start_new_strategy, - start_show_trades, start_test_pairlist, start_trading, - start_webserver) +from freqtrade.commands import (start_convert_data, start_convert_trades, start_create_userdir, + start_download_data, start_hyperopt_list, start_hyperopt_show, + start_install_ui, start_list_data, start_list_exchanges, + start_list_markets, start_list_strategies, start_list_timeframes, + start_new_strategy, start_show_trades, start_test_pairlist, + start_trading, start_webserver) from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) from freqtrade.configuration import setup_utils_configuration @@ -759,6 +759,22 @@ def test_download_data_trades(mocker, caplog): assert convert_mock.call_count == 1 +def test_start_convert_trades(mocker, caplog): + convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv', + MagicMock(return_value=[])) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + args = [ + "trades-to-ohlcv", + "--exchange", "kraken", + "--pairs", "ETH/BTC", "XRP/BTC", + ] + start_convert_trades(get_args(args)) + assert convert_mock.call_count == 1 + + def test_start_list_strategies(mocker, caplog, capsys): args = [ From 178db516bf79dbaa37ca2ef9b779d77ed8850f75 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Sep 2021 19:48:56 +0200 Subject: [PATCH 157/239] Add documentation for trade-to-ohlcv --- docs/data-download.md | 55 +++++++++++++++++++++++++++++++++ freqtrade/commands/arguments.py | 15 +++++---- 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index 0ca86b0d3..5f605c404 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -204,6 +204,61 @@ It'll also remove original jsongz data files (`--erase` parameter). freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase ``` +### Sub-command trades to ohlcv + +When you need to use `--dl-trades` (kraken only) to download data, conversion of trades data to ohlcv data is the last step. +This command will allow you to repeat this last step for additional timeframes without re-downloading the data. + +``` +usage: freqtrade trades-to-ohlcv [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [-p PAIRS [PAIRS ...]] + [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]] + [--exchange EXCHANGE] + [--data-format-ohlcv {json,jsongz,hdf5}] + [--data-format-trades {json,jsongz,hdf5}] + +optional arguments: + -h, --help show this help message and exit + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Limit command to these pairs. Pairs are space- + separated. + -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...] + Specify which tickers to download. Space-separated + list. Default: `1m 5m`. + --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no + config is provided. + --data-format-ohlcv {json,jsongz,hdf5} + Storage format for downloaded candle (OHLCV) data. + (default: `json`). + --data-format-trades {json,jsongz,hdf5} + Storage format for downloaded trades data. (default: + `jsongz`). + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + +``` + +#### Example trade-to-ohlcv conversion + +``` bash +freqtrade trades-to-ohlcv --exchange kraken -t 5m 1h 1d --pairs BTC/EUR ETH/EUR +``` + ### Sub-command list-data You can get a list of downloaded data using the `list-data` sub-command. diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 2fadf047e..9643705a5 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -172,14 +172,13 @@ class Arguments: self._build_args(optionlist=['version'], parser=self.parser) from freqtrade.commands import (start_backtesting, start_convert_data, start_convert_trades, - start_create_userdir, - start_download_data, start_edge, start_hyperopt, - start_hyperopt_list, start_hyperopt_show, start_install_ui, - start_list_data, start_list_exchanges, start_list_markets, - start_list_strategies, start_list_timeframes, - start_new_config, start_new_strategy, start_plot_dataframe, - start_plot_profit, start_show_trades, start_test_pairlist, - start_trading, start_webserver) + start_create_userdir, start_download_data, start_edge, + start_hyperopt, start_hyperopt_list, start_hyperopt_show, + start_install_ui, start_list_data, start_list_exchanges, + start_list_markets, start_list_strategies, + start_list_timeframes, start_new_config, start_new_strategy, + start_plot_dataframe, start_plot_profit, start_show_trades, + start_test_pairlist, start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added From bd27993e797c8d83d54ac76b278a9779e9c4eee5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Sep 2021 06:42:42 +0200 Subject: [PATCH 158/239] Add documentation segment about indicator libraries --- docs/strategy-customization.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 110365208..0bfc0a2f6 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -122,6 +122,16 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame Look into the [user_data/strategies/sample_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_strategy.py). Then uncomment indicators you need. +#### Indicator libraries + +Out of the box, freqtrade installs the following technical libraries: + +* [ta-lib](http://mrjbq7.github.io/ta-lib/) +* [pandas-ta](https://twopirllc.github.io/pandas-ta/) +* [technical](https://github.com/freqtrade/technical/) + +Additional technical libraries can be installed as necessary, or custom indicators may be written / invented by the strategy author. + ### Strategy startup period Most indicators have an instable startup period, in which they are either not available, or the calculation is incorrect. This can lead to inconsistencies, since Freqtrade does not know how long this instable period should be. From 5f23af580248a86a56d60e050bd5d0a7f5424236 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Sep 2021 07:24:16 +0200 Subject: [PATCH 159/239] Rename update_open_trades to clarify it's only called at startup --- freqtrade/freqtradebot.py | 4 ++-- tests/test_freqtradebot.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3a9b21b7c..bf4742fdc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -139,7 +139,7 @@ class FreqtradeBot(LoggingMixin): # Only update open orders on startup # This will update the database after the initial migration - self.update_open_orders() + self.startup_update_open_orders() def process(self) -> None: """ @@ -237,7 +237,7 @@ class FreqtradeBot(LoggingMixin): open_trades = len(Trade.get_open_trades()) return max(0, self.config['max_open_trades'] - open_trades) - def update_open_orders(self): + def startup_update_open_orders(self): """ Updates open orders based on order list kept in the database. Mainly updates the state of orders - but may also close trades diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 760a9dee7..d312bdb11 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -4033,16 +4033,16 @@ def test_check_for_open_trades(mocker, default_conf, fee): @pytest.mark.usefixtures("init_persistence") -def test_update_open_orders(mocker, default_conf, fee, caplog): +def test_startup_update_open_orders(mocker, default_conf, fee, caplog): freqtrade = get_patched_freqtradebot(mocker, default_conf) create_mock_trades(fee) - freqtrade.update_open_orders() + freqtrade.startup_update_open_orders() assert not log_has_re(r"Error updating Order .*", caplog) caplog.clear() freqtrade.config['dry_run'] = False - freqtrade.update_open_orders() + freqtrade.startup_update_open_orders() assert log_has_re(r"Error updating Order .*", caplog) caplog.clear() @@ -4053,7 +4053,7 @@ def test_update_open_orders(mocker, default_conf, fee, caplog): 'status': 'closed', }) mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=matching_buy_order) - freqtrade.update_open_orders() + freqtrade.startup_update_open_orders() # Only stoploss and sell orders are kept open assert len(Order.get_open_orders()) == 2 From 15df5fd9c5c8533b14809147570e2a66d79241fe Mon Sep 17 00:00:00 2001 From: Robert Davey Date: Fri, 1 Oct 2021 13:49:16 +0100 Subject: [PATCH 160/239] Fix pair_candles to point to correct API call pair_candles pointed to available_pairs RPC call instead of pair_candles --- scripts/rest_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/rest_client.py b/scripts/rest_client.py index ece0a253e..713b398c3 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -312,7 +312,7 @@ class FtRestClient(): :param limit: Limit result to the last n candles. :return: json object """ - return self._get("available_pairs", params={ + return self._get("pair_candles", params={ "pair": pair, "timeframe": timeframe, "limit": limit, From f69cb39a170a4a223de4f154c902c69903188a36 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 Oct 2021 19:26:51 +0200 Subject: [PATCH 161/239] Fix missing comma in kucoin template closes #5646 --- freqtrade/templates/subtemplates/exchange_kucoin.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/templates/subtemplates/exchange_kucoin.j2 b/freqtrade/templates/subtemplates/exchange_kucoin.j2 index f9dfff663..9882c51c7 100644 --- a/freqtrade/templates/subtemplates/exchange_kucoin.j2 +++ b/freqtrade/templates/subtemplates/exchange_kucoin.j2 @@ -4,7 +4,7 @@ "secret": "{{ exchange_secret }}", "password": "{{ exchange_key_password }}", "ccxt_config": { - "enableRateLimit": true + "enableRateLimit": true, "rateLimit": 200 }, "ccxt_async_config": { From dadd134200bdcca66456e0795d92c12841fb8e87 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 17 Sep 2021 02:25:58 -0600 Subject: [PATCH 162/239] changes some tests to use usdt values --- tests/conftest.py | 106 ++++++- tests/conftest_trades_usdt.py | 305 ++++++++++++++++++ tests/test_freqtradebot.py | 575 +++++++++++++++++----------------- 3 files changed, 699 insertions(+), 287 deletions(-) create mode 100644 tests/conftest_trades_usdt.py diff --git a/tests/conftest.py b/tests/conftest.py index c908c0cb0..c1032a215 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,8 @@ from freqtrade.resolvers import ExchangeResolver from freqtrade.worker import Worker from tests.conftest_trades import (mock_trade_1, mock_trade_2, mock_trade_3, mock_trade_4, mock_trade_5, mock_trade_6) +from tests.conftest_trades_usdt import (mock_trade_usdt_1, mock_trade_usdt_2, mock_trade_usdt_3, + mock_trade_usdt_4, mock_trade_usdt_5, mock_trade_usdt_6) logging.getLogger('').setLevel(logging.INFO) @@ -227,6 +229,39 @@ def create_mock_trades(fee, use_db: bool = True): Trade.query.session.flush() +def create_mock_trades_usdt(fee, use_db: bool = True): + """ + Create some fake trades ... + """ + def add_trade(trade): + if use_db: + Trade.query.session.add(trade) + else: + LocalTrade.add_bt_trade(trade) + + # Simulate dry_run entries + trade = mock_trade_usdt_1(fee) + add_trade(trade) + + trade = mock_trade_usdt_2(fee) + add_trade(trade) + + trade = mock_trade_usdt_3(fee) + add_trade(trade) + + trade = mock_trade_usdt_4(fee) + add_trade(trade) + + trade = mock_trade_usdt_5(fee) + add_trade(trade) + + trade = mock_trade_usdt_6(fee) + add_trade(trade) + + if use_db: + Trade.query.session.flush() + + @pytest.fixture(autouse=True) def patch_coingekko(mocker) -> None: """ @@ -303,7 +338,8 @@ def get_default_conf(testdatadir): "ETH/BTC", "LTC/BTC", "XRP/BTC", - "NEO/BTC" + "NEO/BTC", + "ADA/USDT" ], "pair_blacklist": [ "DOGE/BTC", @@ -372,6 +408,33 @@ def ticker_sell_down(): }) +@pytest.fixture +def ticker_usdt(): + return MagicMock(return_value={ + 'bid': 1.99, + 'ask': 2.0, + 'last': 1.99, + }) + + +@pytest.fixture +def ticker_usdt_sell_up(): + return MagicMock(return_value={ + 'bid': 2.19, + 'ask': 2.2, + 'last': 2.19, + }) + + +@pytest.fixture +def ticker_usdt_sell_down(): + return MagicMock(return_value={ + 'bid': 2.01, + 'ask': 2.0, + 'last': 2.01, + }) + + @pytest.fixture def markets(): return get_markets() @@ -406,6 +469,31 @@ def get_markets(): }, 'info': {}, }, + 'ADA/USDT': { + 'id': 'ethbtc', + 'symbol': 'ADA/USDT', + 'base': 'USDT', + 'quote': 'ADA', + 'active': True, + 'precision': { + 'price': 8, + 'amount': 8, + 'cost': 8, + }, + 'lot': 0.00000001, + 'limits': { + 'amount': { + 'min': 0.01, + 'max': 1000, + }, + 'price': 500000, + 'cost': { + 'min': 0.0001, + 'max': 500000, + }, + }, + 'info': {}, + }, 'TKN/BTC': { 'id': 'tknbtc', 'symbol': 'TKN/BTC', @@ -1821,6 +1909,22 @@ def open_trade(): ) +@pytest.fixture(scope="function") +def open_trade_usdt(): + return Trade( + pair='ADA/USDT', + open_rate=2.0, + exchange='binance', + open_order_id='123456789', + amount=30.0, + fee_open=0.0, + fee_close=0.0, + stake_amount=60.0, + open_date=arrow.utcnow().shift(minutes=-601).datetime, + is_open=True + ) + + @pytest.fixture def saved_hyperopt_results(): hyperopt_res = [ diff --git a/tests/conftest_trades_usdt.py b/tests/conftest_trades_usdt.py new file mode 100644 index 000000000..1a03f0381 --- /dev/null +++ b/tests/conftest_trades_usdt.py @@ -0,0 +1,305 @@ +from datetime import datetime, timedelta, timezone + +from freqtrade.persistence.models import Order, Trade + + +MOCK_TRADE_COUNT = 6 + + +def mock_order_usdt_1(): + return { + 'id': '1234', + 'symbol': 'ADA/USDT', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 2.0, + 'amount': 10.0, + 'filled': 10.0, + 'remaining': 0.0, + } + + +def mock_trade_usdt_1(fee): + trade = Trade( + pair='ADA/USDT', + stake_amount=20.0, + amount=10.0, + amount_requested=10.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + is_open=True, + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=17), + open_rate=2.0, + exchange='binance', + open_order_id='dry_run_buy_12345', + strategy='StrategyTestV2', + timeframe=5, + ) + o = Order.parse_from_ccxt_object(mock_order_usdt_1(), 'ADA/USDT', 'buy') + trade.orders.append(o) + return trade + + +def mock_order_usdt_2(): + return { + 'id': '1235', + 'symbol': 'ETC/USDT', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 2.0, + 'amount': 100.0, + 'filled': 100.0, + 'remaining': 0.0, + } + + +def mock_order_usdt_2_sell(): + return { + 'id': '12366', + 'symbol': 'ETC/USDT', + 'status': 'closed', + 'side': 'sell', + 'type': 'limit', + 'price': 2.05, + 'amount': 100.0, + 'filled': 100.0, + 'remaining': 0.0, + } + + +def mock_trade_usdt_2(fee): + """ + Closed trade... + """ + trade = Trade( + pair='ETC/USDT', + stake_amount=200.0, + amount=100.0, + amount_requested=100.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=2.0, + close_rate=2.05, + close_profit=5.0, + close_profit_abs=3.9875, + exchange='binance', + is_open=False, + open_order_id='dry_run_sell_12345', + strategy='StrategyTestV2', + timeframe=5, + sell_reason='sell_signal', + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), + close_date=datetime.now(tz=timezone.utc) - timedelta(minutes=2), + ) + o = Order.parse_from_ccxt_object(mock_order_usdt_2(), 'ETC/USDT', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_usdt_2_sell(), 'ETC/USDT', 'sell') + trade.orders.append(o) + return trade + + +def mock_order_usdt_3(): + return { + 'id': '41231a12a', + 'symbol': 'XRP/USDT', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 1.0, + 'amount': 30.0, + 'filled': 30.0, + 'remaining': 0.0, + } + + +def mock_order_usdt_3_sell(): + return { + 'id': '41231a666a', + 'symbol': 'XRP/USDT', + 'status': 'closed', + 'side': 'sell', + 'type': 'stop_loss_limit', + 'price': 1.1, + 'average': 1.1, + 'amount': 30.0, + 'filled': 30.0, + 'remaining': 0.0, + } + + +def mock_trade_usdt_3(fee): + """ + Closed trade + """ + trade = Trade( + pair='XRP/USDT', + stake_amount=30.0, + amount=30.0, + amount_requested=30.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=1.0, + close_rate=1.1, + close_profit=10.0, + close_profit_abs=9.8425, + exchange='binance', + is_open=False, + strategy='StrategyTestV2', + timeframe=5, + sell_reason='roi', + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), + close_date=datetime.now(tz=timezone.utc), + ) + o = Order.parse_from_ccxt_object(mock_order_usdt_3(), 'XRP/USDT', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_usdt_3_sell(), 'XRP/USDT', 'sell') + trade.orders.append(o) + return trade + + +def mock_order_usdt_4(): + return { + 'id': 'prod_buy_12345', + 'symbol': 'ETC/USDT', + 'status': 'open', + 'side': 'buy', + 'type': 'limit', + 'price': 2.0, + 'amount': 10.0, + 'filled': 0.0, + 'remaining': 30.0, + } + + +def mock_trade_usdt_4(fee): + """ + Simulate prod entry + """ + trade = Trade( + pair='ETC/USDT', + stake_amount=20.0, + amount=10.0, + amount_requested=10.01, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=14), + is_open=True, + open_rate=2.0, + exchange='binance', + open_order_id='prod_buy_12345', + strategy='StrategyTestV2', + timeframe=5, + ) + o = Order.parse_from_ccxt_object(mock_order_usdt_4(), 'ETC/USDT', 'buy') + trade.orders.append(o) + return trade + + +def mock_order_usdt_5(): + return { + 'id': 'prod_buy_3455', + 'symbol': 'XRP/USDT', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 2.0, + 'amount': 10.0, + 'filled': 10.0, + 'remaining': 0.0, + } + + +def mock_order_usdt_5_stoploss(): + return { + 'id': 'prod_stoploss_3455', + 'symbol': 'XRP/USDT', + 'status': 'open', + 'side': 'sell', + 'type': 'stop_loss_limit', + 'price': 2.0, + 'amount': 10.0, + 'filled': 0.0, + 'remaining': 30.0, + } + + +def mock_trade_usdt_5(fee): + """ + Simulate prod entry with stoploss + """ + trade = Trade( + pair='XRP/USDT', + stake_amount=20.0, + amount=10.0, + amount_requested=10.01, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=12), + is_open=True, + open_rate=2.0, + exchange='binance', + strategy='SampleStrategy', + stoploss_order_id='prod_stoploss_3455', + timeframe=5, + ) + o = Order.parse_from_ccxt_object(mock_order_usdt_5(), 'XRP/USDT', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_usdt_5_stoploss(), 'XRP/USDT', 'stoploss') + trade.orders.append(o) + return trade + + +def mock_order_usdt_6(): + return { + 'id': 'prod_buy_6', + 'symbol': 'LTC/USDT', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 10.0, + 'amount': 2.0, + 'filled': 2.0, + 'remaining': 0.0, + } + + +def mock_order_usdt_6_sell(): + return { + 'id': 'prod_sell_6', + 'symbol': 'LTC/USDT', + 'status': 'open', + 'side': 'sell', + 'type': 'limit', + 'price': 12.0, + 'amount': 2.0, + 'filled': 0.0, + 'remaining': 2.0, + } + + +def mock_trade_usdt_6(fee): + """ + Simulate prod entry with open sell order + """ + trade = Trade( + pair='LTC/USDT', + stake_amount=20.0, + amount=2.0, + amount_requested=2.0, + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=5), + fee_open=fee.return_value, + fee_close=fee.return_value, + is_open=True, + open_rate=10.0, + exchange='binance', + strategy='SampleStrategy', + open_order_id="prod_sell_6", + timeframe=5, + ) + o = Order.parse_from_ccxt_object(mock_order_usdt_6(), 'LTC/USDT', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_usdt_6_sell(), 'LTC/USDT', 'sell') + trade.orders.append(o) + return trade diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index d312bdb11..7ddb90657 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -23,9 +23,9 @@ from freqtrade.worker import Worker from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker, log_has, log_has_re, patch_edge, patch_exchange, patch_get_signal, patch_wallet, patch_whitelist) -from tests.conftest_trades import (MOCK_TRADE_COUNT, mock_order_1, mock_order_2, mock_order_2_sell, - mock_order_3, mock_order_3_sell, mock_order_4, - mock_order_5_stoploss, mock_order_6_sell) +from tests.conftest_trades import ( + MOCK_TRADE_COUNT, mock_order_1, mock_order_2, mock_order_2_sell, mock_order_3, + mock_order_3_sell, mock_order_4, mock_order_5_stoploss, mock_order_6_sell) def patch_RPCManager(mocker) -> MagicMock: @@ -135,14 +135,14 @@ def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None: (True, 0.0027, 3, 0.5, [0.001, 0.001, 0.000673]), (True, 0.0022, 3, 1, [0.001, 0.001, 0.0]), ]) -def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_buy_order_open, +def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_buy_order_usdt_open, amend_last, wallet, max_open, lsamr, expected) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - create_order=MagicMock(return_value=limit_buy_order_open), + create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee ) default_conf['dry_run_wallet'] = wallet @@ -155,7 +155,7 @@ def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_b for i in range(0, max_open): if expected[i] is not None: - limit_buy_order_open['id'] = str(i) + limit_buy_order_usdt_open['id'] = str(i) result = freqtrade.wallets.get_trade_stake_amount('ETH/BTC') assert pytest.approx(result) == expected[i] freqtrade.execute_entry('ETH/BTC', result) @@ -198,7 +198,7 @@ def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: # Override strategy stoploss (0.85, True) ]) -def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, +def test_edge_overrides_stoploss(limit_buy_order_usdt, fee, caplog, mocker, buy_price_mult, ignore_strat_sl, edge_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -209,7 +209,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, # Thus, if price falls 21%, stoploss should be triggered # # mocking the ticker: price is falling ... - buy_price = limit_buy_order['price'] + buy_price = limit_buy_order_usdt['price'] mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ @@ -221,14 +221,14 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, ) ############################################# - # Create a trade with "limit_buy_order" price + # Create a trade with "limit_buy_order_usdt" price freqtrade = FreqtradeBot(edge_conf) freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.enter_positions() trade = Trade.query.first() - trade.update(limit_buy_order) + trade.update(limit_buy_order_usdt) ############################################# # stoploss shoud be hit @@ -270,7 +270,7 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, fee) -> None: assert Trade.total_open_trades_stakes() == 1.97502e-03 -def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> None: +def test_create_trade(default_conf, ticker, limit_buy_order_usdt, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -294,15 +294,15 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> Non assert trade.exchange == 'binance' # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + trade.update(limit_buy_order_usdt) - assert trade.open_rate == 0.00001099 - assert trade.amount == 90.99181073 + assert trade.open_rate == 2.0 + assert trade.amount == 30.0 assert whitelist == default_conf['exchange']['pair_whitelist'] -def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, +def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order_usdt, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -326,12 +326,12 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, (UNLIMITED_STAKE_AMOUNT, False, True, 0), ]) def test_create_trade_minimal_amount( - default_conf, ticker, limit_buy_order_open, fee, mocker, + default_conf, ticker, limit_buy_order_usdt_open, fee, mocker, stake_amount, create, amount_enough, max_open_trades, caplog ) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - buy_mock = MagicMock(return_value=limit_buy_order_open) + buy_mock = MagicMock(return_value=limit_buy_order_usdt_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, @@ -363,14 +363,14 @@ def test_create_trade_minimal_amount( (["ETH/BTC"], 1), # No pairs left ([], 0), # No pairs in whitelist ]) -def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_open, fee, +def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_usdt_open, fee, whitelist, positions, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - create_order=MagicMock(return_value=limit_buy_order_open), + create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) default_conf['exchange']['pair_whitelist'] = whitelist @@ -390,14 +390,14 @@ def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_ope @pytest.mark.usefixtures("init_persistence") -def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order, fee, +def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order_usdt, fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - create_order=MagicMock(return_value={'id': limit_buy_order['id']}), + create_order=MagicMock(return_value={'id': limit_buy_order_usdt['id']}), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -459,7 +459,7 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: @pytest.mark.parametrize("max_open", range(0, 5)) @pytest.mark.parametrize("tradable_balance_ratio,modifier", [(1.0, 1), (0.99, 0.8), (0.5, 0.5)]) -def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, limit_buy_order_open, +def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, limit_buy_order_usdt_open, max_open, tradable_balance_ratio, modifier) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -470,7 +470,7 @@ def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, limit_ mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - create_order=MagicMock(return_value=limit_buy_order_open), + create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -484,14 +484,14 @@ def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, limit_ assert len(trades) == max(int(max_open * modifier), 0) -def test_create_trades_preopen(default_conf, ticker, fee, mocker, limit_buy_order_open) -> None: +def test_create_trades_preopen(default_conf, ticker, fee, mocker, limit_buy_order_usdt_open) -> None: patch_RPCManager(mocker) patch_exchange(mocker) default_conf['max_open_trades'] = 4 mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - create_order=MagicMock(return_value=limit_buy_order_open), + create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -503,7 +503,7 @@ def test_create_trades_preopen(default_conf, ticker, fee, mocker, limit_buy_orde assert len(Trade.get_open_trades()) == 2 # Change order_id for new orders - limit_buy_order_open['id'] = '123444' + limit_buy_order_usdt_open['id'] = '123444' # Create 2 new trades using create_trades assert freqtrade.create_trade('ETH/BTC') @@ -513,15 +513,15 @@ def test_create_trades_preopen(default_conf, ticker, fee, mocker, limit_buy_orde assert len(trades) == 4 -def test_process_trade_creation(default_conf, ticker, limit_buy_order, limit_buy_order_open, +def test_process_trade_creation(default_conf, ticker, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - create_order=MagicMock(return_value=limit_buy_order_open), - fetch_order=MagicMock(return_value=limit_buy_order), + create_order=MagicMock(return_value=limit_buy_order_usdt_open), + fetch_order=MagicMock(return_value=limit_buy_order_usdt), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -584,14 +584,14 @@ def test_process_operational_exception(default_conf, ticker, mocker) -> None: assert 'OperationalException' in msg_mock.call_args_list[-1][0][0]['status'] -def test_process_trade_handling(default_conf, ticker, limit_buy_order_open, fee, mocker) -> None: +def test_process_trade_handling(default_conf, ticker, limit_buy_order_usdt_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - create_order=MagicMock(return_value=limit_buy_order_open), - fetch_order=MagicMock(return_value=limit_buy_order_open), + create_order=MagicMock(return_value=limit_buy_order_usdt_open), + fetch_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -609,7 +609,7 @@ def test_process_trade_handling(default_conf, ticker, limit_buy_order_open, fee, assert len(trades) == 1 -def test_process_trade_no_whitelist_pair(default_conf, ticker, limit_buy_order, +def test_process_trade_no_whitelist_pair(default_conf, ticker, limit_buy_order_usdt, fee, mocker) -> None: """ Test process with trade not in pair list """ patch_RPCManager(mocker) @@ -617,8 +617,8 @@ def test_process_trade_no_whitelist_pair(default_conf, ticker, limit_buy_order, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - create_order=MagicMock(return_value={'id': limit_buy_order['id']}), - fetch_order=MagicMock(return_value=limit_buy_order), + create_order=MagicMock(return_value={'id': limit_buy_order_usdt['id']}), + fetch_order=MagicMock(return_value=limit_buy_order_usdt), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -690,7 +690,7 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: assert ("ETH/BTC", default_conf["timeframe"]) in refresh_mock.call_args[0][0] -def test_execute_entry(mocker, default_conf, fee, limit_buy_order, limit_buy_order_open) -> None: +def test_execute_entry(mocker, default_conf, fee, limit_buy_order_usdt, limit_buy_order_usdt_open) -> None: patch_RPCManager(mocker) patch_exchange(mocker) freqtrade = FreqtradeBot(default_conf) @@ -698,14 +698,14 @@ def test_execute_entry(mocker, default_conf, fee, limit_buy_order, limit_buy_ord stake_amount = 2 bid = 0.11 buy_rate_mock = MagicMock(return_value=bid) - buy_mm = MagicMock(return_value=limit_buy_order_open) + buy_mm = MagicMock(return_value=limit_buy_order_usdt_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_rate=buy_rate_mock, fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 + 'bid': 1.9, + 'ask': 2.2, + 'last': 1.9 }), create_order=buy_mm, get_min_pair_stake_amount=MagicMock(return_value=1), @@ -719,7 +719,7 @@ def test_execute_entry(mocker, default_conf, fee, limit_buy_order, limit_buy_ord assert freqtrade.strategy.confirm_trade_entry.call_count == 1 buy_rate_mock.reset_mock() - limit_buy_order_open['id'] = '22' + limit_buy_order_usdt_open['id'] = '22' freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=True) assert freqtrade.execute_entry(pair, stake_amount) assert buy_rate_mock.call_count == 1 @@ -738,7 +738,7 @@ def test_execute_entry(mocker, default_conf, fee, limit_buy_order, limit_buy_ord assert trade.open_order_id == '22' # Test calling with price - limit_buy_order_open['id'] = '33' + limit_buy_order_usdt_open['id'] = '33' fix_price = 0.06 assert freqtrade.execute_entry(pair, stake_amount, fix_price) # Make sure get_rate wasn't called again @@ -751,13 +751,13 @@ def test_execute_entry(mocker, default_conf, fee, limit_buy_order, limit_buy_ord assert call_args['amount'] == round(stake_amount / fix_price, 8) # In case of closed order - limit_buy_order['status'] = 'closed' - limit_buy_order['price'] = 10 - limit_buy_order['cost'] = 100 - limit_buy_order['id'] = '444' + limit_buy_order_usdt['status'] = 'closed' + limit_buy_order_usdt['price'] = 10 + limit_buy_order_usdt['cost'] = 100 + limit_buy_order_usdt['id'] = '444' mocker.patch('freqtrade.exchange.Exchange.create_order', - MagicMock(return_value=limit_buy_order)) + MagicMock(return_value=limit_buy_order_usdt)) assert freqtrade.execute_entry(pair, stake_amount) trade = Trade.query.all()[2] assert trade @@ -766,15 +766,15 @@ def test_execute_entry(mocker, default_conf, fee, limit_buy_order, limit_buy_ord assert trade.stake_amount == 100 # In case of rejected or expired order and partially filled - limit_buy_order['status'] = 'expired' - limit_buy_order['amount'] = 90.99181073 - limit_buy_order['filled'] = 80.99181073 - limit_buy_order['remaining'] = 10.00 - limit_buy_order['price'] = 0.5 - limit_buy_order['cost'] = 40.495905365 - limit_buy_order['id'] = '555' + limit_buy_order_usdt['status'] = 'expired' + limit_buy_order_usdt['amount'] = 30.0 + limit_buy_order_usdt['filled'] = 80.99181073 + limit_buy_order_usdt['remaining'] = 10.00 + limit_buy_order_usdt['price'] = 0.5 + limit_buy_order_usdt['cost'] = 40.495905365 + limit_buy_order_usdt['id'] = '555' mocker.patch('freqtrade.exchange.Exchange.create_order', - MagicMock(return_value=limit_buy_order)) + MagicMock(return_value=limit_buy_order_usdt)) assert freqtrade.execute_entry(pair, stake_amount) trade = Trade.query.all()[3] assert trade @@ -783,8 +783,8 @@ def test_execute_entry(mocker, default_conf, fee, limit_buy_order, limit_buy_ord assert trade.stake_amount == 40.495905365 # Test with custom stake - limit_buy_order['status'] = 'open' - limit_buy_order['id'] = '556' + limit_buy_order_usdt['status'] = 'open' + limit_buy_order_usdt['id'] = '556' freqtrade.strategy.custom_stake_amount = lambda **kwargs: 150.0 assert freqtrade.execute_entry(pair, stake_amount) @@ -793,7 +793,7 @@ def test_execute_entry(mocker, default_conf, fee, limit_buy_order, limit_buy_ord assert trade.stake_amount == 150 # Exception case - limit_buy_order['id'] = '557' + limit_buy_order_usdt['id'] = '557' freqtrade.strategy.custom_stake_amount = lambda **kwargs: 20 / 0 assert freqtrade.execute_entry(pair, stake_amount) trade = Trade.query.all()[5] @@ -801,15 +801,15 @@ def test_execute_entry(mocker, default_conf, fee, limit_buy_order, limit_buy_ord assert trade.stake_amount == 2.0 # In case of the order is rejected and not filled at all - limit_buy_order['status'] = 'rejected' - limit_buy_order['amount'] = 90.99181073 - limit_buy_order['filled'] = 0.0 - limit_buy_order['remaining'] = 90.99181073 - limit_buy_order['price'] = 0.5 - limit_buy_order['cost'] = 0.0 - limit_buy_order['id'] = '66' + limit_buy_order_usdt['status'] = 'rejected' + limit_buy_order_usdt['amount'] = 30.0 + limit_buy_order_usdt['filled'] = 0.0 + limit_buy_order_usdt['remaining'] = 30.0 + limit_buy_order_usdt['price'] = 0.5 + limit_buy_order_usdt['cost'] = 0.0 + limit_buy_order_usdt['id'] = '66' mocker.patch('freqtrade.exchange.Exchange.create_order', - MagicMock(return_value=limit_buy_order)) + MagicMock(return_value=limit_buy_order_usdt)) assert not freqtrade.execute_entry(pair, stake_amount) # Fail to get price... @@ -820,8 +820,8 @@ def test_execute_entry(mocker, default_conf, fee, limit_buy_order, limit_buy_ord # In case of custom entry price mocker.patch('freqtrade.exchange.Exchange.get_rate', return_value=0.50) - limit_buy_order['status'] = 'open' - limit_buy_order['id'] = '5566' + limit_buy_order_usdt['status'] = 'open' + limit_buy_order_usdt['id'] = '5566' freqtrade.strategy.custom_entry_price = lambda **kwargs: 0.508 assert freqtrade.execute_entry(pair, stake_amount) trade = Trade.query.all()[6] @@ -829,8 +829,8 @@ def test_execute_entry(mocker, default_conf, fee, limit_buy_order, limit_buy_ord assert trade.open_rate_requested == 0.508 # In case of custom entry price set to None - limit_buy_order['status'] = 'open' - limit_buy_order['id'] = '5567' + limit_buy_order_usdt['status'] = 'open' + limit_buy_order_usdt['id'] = '5567' freqtrade.strategy.custom_entry_price = lambda **kwargs: None mocker.patch.multiple( @@ -844,8 +844,8 @@ def test_execute_entry(mocker, default_conf, fee, limit_buy_order, limit_buy_ord assert trade.open_rate_requested == 10 # In case of custom entry price not float type - limit_buy_order['status'] = 'open' - limit_buy_order['id'] = '5568' + limit_buy_order_usdt['status'] = 'open' + limit_buy_order_usdt['id'] = '5568' freqtrade.strategy.custom_entry_price = lambda **kwargs: "string price" assert freqtrade.execute_entry(pair, stake_amount) trade = Trade.query.all()[8] @@ -853,16 +853,16 @@ def test_execute_entry(mocker, default_conf, fee, limit_buy_order, limit_buy_ord assert trade.open_rate_requested == 10 -def test_execute_entry_confirm_error(mocker, default_conf, fee, limit_buy_order) -> None: +def test_execute_entry_confirm_error(mocker, default_conf, fee, limit_buy_order_usdt) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 + 'bid': 1.9, + 'ask': 2.2, + 'last': 1.9 }), - create_order=MagicMock(return_value=limit_buy_order), + create_order=MagicMock(return_value=limit_buy_order_usdt), get_rate=MagicMock(return_value=0.11), get_min_pair_stake_amount=MagicMock(return_value=1), get_fee=fee, @@ -873,11 +873,11 @@ def test_execute_entry_confirm_error(mocker, default_conf, fee, limit_buy_order) freqtrade.strategy.confirm_trade_entry = MagicMock(side_effect=ValueError) assert freqtrade.execute_entry(pair, stake_amount) - limit_buy_order['id'] = '222' + limit_buy_order_usdt['id'] = '222' freqtrade.strategy.confirm_trade_entry = MagicMock(side_effect=Exception) assert freqtrade.execute_entry(pair, stake_amount) - limit_buy_order['id'] = '2223' + limit_buy_order_usdt['id'] = '2223' freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=True) assert freqtrade.execute_entry(pair, stake_amount) @@ -885,14 +885,14 @@ def test_execute_entry_confirm_error(mocker, default_conf, fee, limit_buy_order) assert not freqtrade.execute_entry(pair, stake_amount) -def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None: +def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order_usdt) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) - mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order) + mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order_usdt) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', - return_value=limit_buy_order['amount']) + return_value=limit_buy_order_usdt['amount']) stoploss = MagicMock(return_value={'id': 13434334}) mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) @@ -913,20 +913,20 @@ def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, - limit_buy_order, limit_sell_order) -> None: + limit_buy_order_usdt, limit_sell_order_usdt) -> None: stoploss = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 + 'bid': 1.9, + 'ask': 2.2, + 'last': 1.9 }), create_order=MagicMock(side_effect=[ - {'id': limit_buy_order['id']}, - {'id': limit_sell_order['id']}, + {'id': limit_buy_order_usdt['id']}, + {'id': limit_sell_order_usdt['id']}, ]), get_fee=fee, ) @@ -993,7 +993,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, 'type': 'stop_loss_limit', 'price': 3, 'average': 2, - 'amount': limit_buy_order['amount'], + 'amount': limit_buy_order_usdt['amount'], }) mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', stoploss_order_hit) assert freqtrade.handle_stoploss_on_exchange(trade) is True @@ -1033,20 +1033,20 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, - limit_buy_order, limit_sell_order) -> None: + limit_buy_order_usdt, limit_sell_order_usdt) -> None: # Sixth case: stoploss order was cancelled but couldn't create new one patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 + 'bid': 1.9, + 'ask': 2.2, + 'last': 1.9 }), create_order=MagicMock(side_effect=[ - {'id': limit_buy_order['id']}, - {'id': limit_sell_order['id']}, + {'id': limit_buy_order_usdt['id']}, + {'id': limit_sell_order_usdt['id']}, ]), get_fee=fee, ) @@ -1072,19 +1072,19 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, - limit_buy_order_open, limit_sell_order): + limit_buy_order_usdt_open, limit_sell_order_usdt): rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) create_order_mock = MagicMock(side_effect=[ - limit_buy_order_open, - {'id': limit_sell_order['id']} + limit_buy_order_usdt_open, + {'id': limit_sell_order_usdt['id']} ]) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 + 'bid': 1.9, + 'ask': 2.2, + 'last': 1.9 }), create_order=create_order_mock, get_fee=fee, @@ -1120,20 +1120,20 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, def test_create_stoploss_order_insufficient_funds(mocker, default_conf, caplog, fee, - limit_buy_order_open, limit_sell_order): - sell_mock = MagicMock(return_value={'id': limit_sell_order['id']}) + limit_buy_order_usdt_open, limit_sell_order_usdt): + sell_mock = MagicMock(return_value={'id': limit_sell_order_usdt['id']}) freqtrade = get_patched_freqtradebot(mocker, default_conf) mock_insuf = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_insufficient_funds') mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 + 'bid': 1.9, + 'ask': 2.2, + 'last': 1.9 }), create_order=MagicMock(side_effect=[ - limit_buy_order_open, + limit_buy_order_usdt_open, sell_mock, ]), get_fee=fee, @@ -1164,20 +1164,20 @@ def test_create_stoploss_order_insufficient_funds(mocker, default_conf, caplog, @pytest.mark.usefixtures("init_persistence") def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, - limit_buy_order, limit_sell_order) -> None: + limit_buy_order_usdt, limit_sell_order_usdt) -> None: # When trailing stoploss is set stoploss = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 + 'bid': 2.19, + 'ask': 2.2, + 'last': 2.19 }), create_order=MagicMock(side_effect=[ - {'id': limit_buy_order['id']}, - {'id': limit_sell_order['id']}, + {'id': limit_buy_order_usdt['id']}, + {'id': limit_sell_order_usdt['id']}, ]), get_fee=fee, ) @@ -1219,7 +1219,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, 'price': 3, 'average': 2, 'info': { - 'stopPrice': '0.000011134' + 'stopPrice': '2.0805' } }) @@ -1230,11 +1230,14 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, assert freqtrade.handle_stoploss_on_exchange(trade) is False # price jumped 2x - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ - 'bid': 0.00002344, - 'ask': 0.00002346, - 'last': 0.00002344 - })) + mocker.patch( + 'freqtrade.exchange.Exchange.fetch_ticker', + MagicMock(return_value={ + 'bid': 4.38, + 'ask': 4.4, + 'last': 4.38 + }) + ) cancel_order_mock = MagicMock() stoploss_order_mock = MagicMock(return_value={'id': 13434334}) @@ -1248,7 +1251,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, stoploss_order_mock.assert_not_called() assert freqtrade.handle_trade(trade) is False - assert trade.stop_loss == 0.00002346 * 0.95 + assert trade.stop_loss == 4.4 * 0.95 # setting stoploss_on_exchange_interval to 0 seconds freqtrade.strategy.order_types['stoploss_on_exchange_interval'] = 0 @@ -1256,22 +1259,24 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, assert freqtrade.handle_stoploss_on_exchange(trade) is False cancel_order_mock.assert_called_once_with(100, 'ETH/BTC') - stoploss_order_mock.assert_called_once_with(amount=85.32423208, - pair='ETH/BTC', - order_types=freqtrade.strategy.order_types, - stop_price=0.00002346 * 0.95) + stoploss_order_mock.assert_called_once_with( + amount=30.0, + pair='ETH/BTC', + order_types=freqtrade.strategy.order_types, + stop_price=4.4 * 0.95 + ) # price fell below stoploss, so dry-run sells trade. mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ - 'bid': 0.00002144, - 'ask': 0.00002146, - 'last': 0.00002144 + 'bid': 4.1712, + 'ask': 4.1921, + 'last': 4.1712 })) assert freqtrade.handle_trade(trade) is True def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, caplog, - limit_buy_order, limit_sell_order) -> None: + limit_buy_order_usdt, limit_sell_order_usdt) -> None: # When trailing stoploss is set stoploss = MagicMock(return_value={'id': 13434334}) patch_exchange(mocker) @@ -1279,13 +1284,13 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 + 'bid': 1.9, + 'ask': 2.2, + 'last': 1.9 }), create_order=MagicMock(side_effect=[ - {'id': limit_buy_order['id']}, - {'id': limit_sell_order['id']}, + {'id': limit_buy_order_usdt['id']}, + {'id': limit_sell_order_usdt['id']}, ]), get_fee=fee, ) @@ -1347,20 +1352,20 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c @pytest.mark.usefixtures("init_persistence") def test_handle_stoploss_on_exchange_custom_stop(mocker, default_conf, fee, - limit_buy_order, limit_sell_order) -> None: + limit_buy_order_usdt, limit_sell_order_usdt) -> None: # When trailing stoploss is set stoploss = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 + 'bid': 1.9, + 'ask': 2.2, + 'last': 1.9 }), create_order=MagicMock(side_effect=[ - {'id': limit_buy_order['id']}, - {'id': limit_sell_order['id']}, + {'id': limit_buy_order_usdt['id']}, + {'id': limit_sell_order_usdt['id']}, ]), get_fee=fee, ) @@ -1413,9 +1418,9 @@ def test_handle_stoploss_on_exchange_custom_stop(mocker, default_conf, fee, # price jumped 2x mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ - 'bid': 0.00002344, - 'ask': 0.00002346, - 'last': 0.00002344 + 'bid': 4.38, + 'ask': 4.4, + 'last': 4.38 })) cancel_order_mock = MagicMock() @@ -1454,7 +1459,7 @@ def test_handle_stoploss_on_exchange_custom_stop(mocker, default_conf, fee, def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, - limit_buy_order, limit_sell_order) -> None: + limit_buy_order_usdt, limit_sell_order_usdt) -> None: # When trailing stoploss is set stoploss = MagicMock(return_value={'id': 13434334}) @@ -1467,13 +1472,13 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 + 'bid': 1.9, + 'ask': 2.2, + 'last': 1.9 }), create_order=MagicMock(side_effect=[ - {'id': limit_buy_order['id']}, - {'id': limit_sell_order['id']}, + {'id': limit_buy_order_usdt['id']}, + {'id': limit_sell_order_usdt['id']}, ]), get_fee=fee, stoploss=stoploss, @@ -1515,7 +1520,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, 'price': 3, 'average': 2, 'info': { - 'stopPrice': '0.000009384' + 'stopPrice': '2.178' } }) @@ -1524,7 +1529,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, # stoploss initially at 20% as edge dictated it. assert freqtrade.handle_trade(trade) is False assert freqtrade.handle_stoploss_on_exchange(trade) is False - assert trade.stop_loss == 0.000009384 + assert trade.stop_loss == 2.178 cancel_order_mock = MagicMock() stoploss_order_mock = MagicMock() @@ -1533,16 +1538,16 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, # price goes down 5% mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ - 'bid': 0.00001172 * 0.95, - 'ask': 0.00001173 * 0.95, - 'last': 0.00001172 * 0.95 + 'bid': 1.9 * 0.95, + 'ask': 2.2 * 0.95, + 'last': 1.9 * 0.95 })) assert freqtrade.handle_trade(trade) is False assert freqtrade.handle_stoploss_on_exchange(trade) is False # stoploss should remain the same - assert trade.stop_loss == 0.000009384 + assert trade.stop_loss == 2.178 # stoploss on exchange should not be canceled cancel_order_mock.assert_not_called() @@ -1589,14 +1594,14 @@ def test_enter_positions(mocker, default_conf, return_value, side_effect, assert mock_ct.call_count == len(default_conf['exchange']['pair_whitelist']) -def test_exit_positions(mocker, default_conf, limit_buy_order, caplog) -> None: +def test_exit_positions(mocker, default_conf, limit_buy_order_usdt, caplog) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) - mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order) + mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order_usdt) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', - return_value=limit_buy_order['amount']) + return_value=limit_buy_order_usdt['amount']) trade = MagicMock() trade.open_order_id = '123' @@ -1606,7 +1611,7 @@ def test_exit_positions(mocker, default_conf, limit_buy_order, caplog) -> None: assert n == 0 # Test amount not modified by fee-logic assert not log_has( - 'Applying fee to amount for Trade {} from 90.99181073 to 90.81'.format(trade), caplog + 'Applying fee to amount for Trade {} from 30.0 to 90.81'.format(trade), caplog ) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=90.81) @@ -1615,9 +1620,9 @@ def test_exit_positions(mocker, default_conf, limit_buy_order, caplog) -> None: assert n == 0 -def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog) -> None: +def test_exit_positions_exception(mocker, default_conf, limit_buy_order_usdt, caplog) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order) + mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order_usdt) trade = MagicMock() trade.open_order_id = None @@ -1635,14 +1640,14 @@ def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog) assert log_has('Unable to sell trade ETH/BTC: ', caplog) -def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> None: +def test_update_trade_state(mocker, default_conf, limit_buy_order_usdt, caplog) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) - mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order) + mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order_usdt) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', - return_value=limit_buy_order['amount']) + return_value=limit_buy_order_usdt['amount']) trade = Trade( open_order_id=123, @@ -1662,7 +1667,7 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No assert not log_has_re(r'Applying fee to .*', caplog) caplog.clear() assert trade.open_order_id is None - assert trade.amount == limit_buy_order['amount'] + assert trade.amount == limit_buy_order_usdt['amount'] trade.open_order_id = '123' mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=90.81) @@ -1681,10 +1686,10 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No @pytest.mark.parametrize('initial_amount,has_rounding_fee', [ - (90.99181073 + 1e-14, True), + (30.0 + 1e-14, True), (8.0, False) ]) -def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_buy_order, fee, +def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_buy_order_usdt, fee, mocker, initial_amount, has_rounding_fee, caplog): trades_for_order[0]['amount'] = initial_amount mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) @@ -1704,17 +1709,17 @@ def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_ open_order_id="123456", is_open=True, ) - freqtrade.update_trade_state(trade, '123456', limit_buy_order) + freqtrade.update_trade_state(trade, '123456', limit_buy_order_usdt) assert trade.amount != amount - assert trade.amount == limit_buy_order['amount'] + assert trade.amount == limit_buy_order_usdt['amount'] if has_rounding_fee: assert log_has_re(r'Applying fee on amount for .*', caplog) def test_update_trade_state_exception(mocker, default_conf, - limit_buy_order, caplog) -> None: + limit_buy_order_usdt, caplog) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order) + mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order_usdt) trade = MagicMock() trade.open_order_id = '123' @@ -1745,8 +1750,8 @@ def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None assert log_has(f'Unable to fetch order {trade.open_order_id}: ', caplog) -def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_order_open, - limit_sell_order, mocker): +def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_order_usdt_open, + limit_sell_order_usdt, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) # fetch_order should not be called!! mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError)) @@ -1754,7 +1759,7 @@ def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_orde mocker.patch('freqtrade.wallets.Wallets.update', wallet_mock) patch_exchange(mocker) - amount = limit_sell_order["amount"] + amount = limit_sell_order_usdt["amount"] freqtrade = get_patched_freqtradebot(mocker, default_conf) wallet_mock.reset_mock() trade = Trade( @@ -1768,11 +1773,11 @@ def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_orde open_order_id="123456", is_open=True, ) - order = Order.parse_from_ccxt_object(limit_sell_order_open, 'LTC/ETH', 'sell') + order = Order.parse_from_ccxt_object(limit_sell_order_usdt_open, 'LTC/ETH', 'sell') trade.orders.append(order) assert order.status == 'open' - freqtrade.update_trade_state(trade, trade.open_order_id, limit_sell_order) - assert trade.amount == limit_sell_order['amount'] + freqtrade.update_trade_state(trade, trade.open_order_id, limit_sell_order_usdt) + assert trade.amount == limit_sell_order_usdt['amount'] # Wallet needs to be updated after closing a limit-sell order to reenable buying assert wallet_mock.call_count == 1 assert not trade.is_open @@ -1780,20 +1785,20 @@ def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_orde assert order.status == 'closed' -def test_handle_trade(default_conf, limit_buy_order, limit_sell_order_open, limit_sell_order, +def test_handle_trade(default_conf, limit_buy_order_usdt, limit_sell_order_usdt_open, limit_sell_order_usdt, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 + 'bid': 1.9, + 'ask': 2.2, + 'last': 1.9 }), create_order=MagicMock(side_effect=[ - limit_buy_order, - limit_sell_order_open, + limit_buy_order_usdt, + limit_sell_order_usdt_open, ]), get_fee=fee, ) @@ -1806,24 +1811,24 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order_open, limi assert trade time.sleep(0.01) # Race condition fix - trade.update(limit_buy_order) + trade.update(limit_buy_order_usdt) assert trade.is_open is True freqtrade.wallets.update() patch_get_signal(freqtrade, value=(False, True, None)) assert freqtrade.handle_trade(trade) is True - assert trade.open_order_id == limit_sell_order['id'] + assert trade.open_order_id == limit_sell_order_usdt['id'] # Simulate fulfilled LIMIT_SELL order for trade - trade.update(limit_sell_order) + trade.update(limit_sell_order_usdt) - assert trade.close_rate == 0.00001173 - assert trade.close_profit == 0.06201058 - assert trade.calc_profit() == 0.00006217 + assert trade.close_rate == 2.2 + assert trade.close_profit == 0.09451372 + assert trade.calc_profit() == 5.685 assert trade.close_date is not None -def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order_open, +def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order_usdt_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -1831,7 +1836,7 @@ def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order_open, 'freqtrade.exchange.Exchange', fetch_ticker=ticker, create_order=MagicMock(side_effect=[ - limit_buy_order_open, + limit_buy_order_usdt_open, {'id': 1234553382}, ]), get_fee=fee, @@ -1878,7 +1883,7 @@ def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order_open, assert freqtrade.handle_trade(trades[0]) is True -def test_handle_trade_roi(default_conf, ticker, limit_buy_order_open, +def test_handle_trade_roi(default_conf, ticker, limit_buy_order_usdt_open, fee, mocker, caplog) -> None: caplog.set_level(logging.DEBUG) @@ -1887,7 +1892,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order_open, 'freqtrade.exchange.Exchange', fetch_ticker=ticker, create_order=MagicMock(side_effect=[ - limit_buy_order_open, + limit_buy_order_usdt_open, {'id': 1234553382}, ]), get_fee=fee, @@ -1913,8 +1918,8 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order_open, caplog) -def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_open, - limit_sell_order_open, fee, mocker, caplog) -> None: +def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_usdt_open, + limit_sell_order_usdt_open, fee, mocker, caplog) -> None: # use_sell_signal is True buy default caplog.set_level(logging.DEBUG) patch_RPCManager(mocker) @@ -1922,8 +1927,8 @@ def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_open 'freqtrade.exchange.Exchange', fetch_ticker=ticker, create_order=MagicMock(side_effect=[ - limit_buy_order_open, - limit_sell_order_open, + limit_buy_order_usdt_open, + limit_sell_order_usdt_open, ]), get_fee=fee, ) @@ -1945,14 +1950,14 @@ def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_open caplog) -def test_close_trade(default_conf, ticker, limit_buy_order, limit_buy_order_open, limit_sell_order, +def test_close_trade(default_conf, ticker, limit_buy_order_usdt, limit_buy_order_usdt_open, limit_sell_order_usdt, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - create_order=MagicMock(return_value=limit_buy_order_open), + create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -1964,8 +1969,8 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_buy_order_open trade = Trade.query.first() assert trade - trade.update(limit_buy_order) - trade.update(limit_sell_order) + trade.update(limit_buy_order_usdt) + trade.update(limit_sell_order_usdt) assert trade.is_open is False with pytest.raises(DependencyException, match=r'.*closed trade.*'): @@ -2341,7 +2346,7 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, assert trades[0].fee_open == fee() -def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocker, caplog) -> None: +def test_check_handle_timedout_exception(default_conf, ticker, open_trade_usdt, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock() @@ -2359,20 +2364,20 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke ) freqtrade = FreqtradeBot(default_conf) - Trade.query.session.add(open_trade) + Trade.query.session.add(open_trade_usdt) freqtrade.check_handle_timedout() - assert log_has_re(r"Cannot query order for Trade\(id=1, pair=ETH/BTC, amount=90.99181073, " - r"open_rate=0.00001099, open_since=" - f"{open_trade.open_date.strftime('%Y-%m-%d %H:%M:%S')}" + assert log_has_re(r"Cannot query order for Trade\(id=1, pair=ADA/USDT, amount=30.00000000, " + r"open_rate=2.00000000, open_since=" + f"{open_trade_usdt.open_date.strftime('%Y-%m-%d %H:%M:%S')}" r"\) due to Traceback \(most recent call last\):\n*", caplog) -def test_handle_cancel_enter(mocker, caplog, default_conf, limit_buy_order) -> None: +def test_handle_cancel_enter(mocker, caplog, default_conf, limit_buy_order_usdt) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - cancel_buy_order = deepcopy(limit_buy_order) + cancel_buy_order = deepcopy(limit_buy_order_usdt) cancel_buy_order['status'] = 'canceled' del cancel_buy_order['filled'] @@ -2385,30 +2390,30 @@ def test_handle_cancel_enter(mocker, caplog, default_conf, limit_buy_order) -> N trade = MagicMock() trade.pair = 'LTC/USDT' trade.open_rate = 200 - limit_buy_order['filled'] = 0.0 - limit_buy_order['status'] = 'open' + limit_buy_order_usdt['filled'] = 0.0 + limit_buy_order_usdt['status'] = 'open' reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) + assert freqtrade.handle_cancel_enter(trade, limit_buy_order_usdt, reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() caplog.clear() - limit_buy_order['filled'] = 0.01 - assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) + limit_buy_order_usdt['filled'] = 0.01 + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order_usdt, reason) assert cancel_order_mock.call_count == 0 assert log_has_re("Order .* for .* not cancelled, as the filled amount.* unsellable.*", caplog) caplog.clear() cancel_order_mock.reset_mock() - limit_buy_order['filled'] = 2 - assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) + limit_buy_order_usdt['filled'] = 2 + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order_usdt, reason) assert cancel_order_mock.call_count == 1 # Order remained open for some reason (cancel failed) cancel_buy_order['status'] = 'open' cancel_order_mock = MagicMock(return_value=cancel_buy_order) mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock) - assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order_usdt, reason) assert log_has_re(r"Order .* for .* not cancelled.", caplog) @@ -2439,7 +2444,7 @@ def test_handle_cancel_enter_exchanges(mocker, caplog, default_conf, 'String Return value', 123 ]) -def test_handle_cancel_enter_corder_empty(mocker, default_conf, limit_buy_order, +def test_handle_cancel_enter_corder_empty(mocker, default_conf, limit_buy_order_usdt, cancelorder) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -2455,15 +2460,15 @@ def test_handle_cancel_enter_corder_empty(mocker, default_conf, limit_buy_order, trade = MagicMock() trade.pair = 'LTC/USDT' trade.open_rate = 200 - limit_buy_order['filled'] = 0.0 - limit_buy_order['status'] = 'open' + limit_buy_order_usdt['filled'] = 0.0 + limit_buy_order_usdt['status'] = 'open' reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) + assert freqtrade.handle_cancel_enter(trade, limit_buy_order_usdt, reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() - limit_buy_order['filled'] = 1.0 - assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) + limit_buy_order_usdt['filled'] = 1.0 + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order_usdt, reason) assert cancel_order_mock.call_count == 1 @@ -3021,7 +3026,7 @@ def test_execute_trade_exit_insufficient_funds_error(default_conf, ticker, fee, @pytest.mark.parametrize('profit_only,bid,ask,handle_first,handle_second,sell_type', [ # Enable profit - (True, 0.00001172, 0.00001173, False, True, SellType.SELL_SIGNAL.value), + (True, 1.9, 2.2, False, True, SellType.SELL_SIGNAL.value), # Disable profit (False, 0.00002172, 0.00002173, True, False, SellType.SELL_SIGNAL.value), # Enable loss @@ -3031,7 +3036,7 @@ def test_execute_trade_exit_insufficient_funds_error(default_conf, ticker, fee, (False, 0.00000172, 0.00000173, True, False, SellType.SELL_SIGNAL.value), ]) def test_sell_profit_only( - default_conf, limit_buy_order, limit_buy_order_open, + default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, mocker, profit_only, bid, ask, handle_first, handle_second, sell_type) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -3043,7 +3048,7 @@ def test_sell_profit_only( 'last': bid }), create_order=MagicMock(side_effect=[ - limit_buy_order_open, + limit_buy_order_usdt_open, {'id': 1234553382}, ]), get_fee=fee, @@ -3063,7 +3068,7 @@ def test_sell_profit_only( freqtrade.enter_positions() trade = Trade.query.first() - trade.update(limit_buy_order) + trade.update(limit_buy_order_usdt) freqtrade.wallets.update() patch_get_signal(freqtrade, value=(False, True, None)) assert freqtrade.handle_trade(trade) is handle_first @@ -3072,10 +3077,8 @@ def test_sell_profit_only( freqtrade.strategy.sell_profit_offset = 0.0 assert freqtrade.handle_trade(trade) is True - assert trade.sell_reason == sell_type - -def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_open, +def test_sell_not_enough_balance(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -3087,7 +3090,7 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_ 'last': 0.00002172 }), create_order=MagicMock(side_effect=[ - limit_buy_order_open, + limit_buy_order_usdt_open, {'id': 1234553382}, ]), get_fee=fee, @@ -3101,7 +3104,7 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_ trade = Trade.query.first() amnt = trade.amount - trade.update(limit_buy_order) + trade.update(limit_buy_order_usdt) patch_get_signal(freqtrade, value=(False, True, None)) mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=trade.amount * 0.985)) @@ -3182,7 +3185,7 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo assert log_has_re(f"Pair {trade.pair} is still locked.*", caplog) -def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, +def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -3194,7 +3197,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order 'last': 0.0000172 }), create_order=MagicMock(side_effect=[ - limit_buy_order_open, + limit_buy_order_usdt_open, {'id': 1234553382}, ]), get_fee=fee, @@ -3208,7 +3211,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order freqtrade.enter_positions() trade = Trade.query.first() - trade.update(limit_buy_order) + trade.update(limit_buy_order_usdt) freqtrade.wallets.update() patch_get_signal(freqtrade, value=(True, True, None)) assert freqtrade.handle_trade(trade) is False @@ -3219,19 +3222,19 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order assert trade.sell_reason == SellType.ROI.value -def test_trailing_stop_loss(default_conf, limit_buy_order_open, limit_buy_order, +def test_trailing_stop_loss(default_conf, limit_buy_order_usdt_open, limit_buy_order_usdt, fee, caplog, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001099, - 'ask': 0.00001099, - 'last': 0.00001099 + 'bid': 2.0, + 'ask': 2.0, + 'last': 2.0 }), create_order=MagicMock(side_effect=[ - limit_buy_order_open, + limit_buy_order_usdt_open, {'id': 1234553382}, ]), get_fee=fee, @@ -3249,9 +3252,9 @@ def test_trailing_stop_loss(default_conf, limit_buy_order_open, limit_buy_order, # Raise ticker above buy price mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ - 'bid': 0.00001099 * 1.5, - 'ask': 0.00001099 * 1.5, - 'last': 0.00001099 * 1.5 + 'bid': 2.0 * 1.5, + 'ask': 2.0 * 1.5, + 'last': 2.0 * 1.5 })) # Stoploss should be adjusted @@ -3260,9 +3263,9 @@ def test_trailing_stop_loss(default_conf, limit_buy_order_open, limit_buy_order, # Price fell mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ - 'bid': 0.00001099 * 1.1, - 'ask': 0.00001099 * 1.1, - 'last': 0.00001099 * 1.1 + 'bid': 2.0 * 1.1, + 'ask': 2.0 * 1.1, + 'last': 2.0 * 1.1 })) caplog.set_level(logging.DEBUG) @@ -3273,9 +3276,9 @@ def test_trailing_stop_loss(default_conf, limit_buy_order_open, limit_buy_order, assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value -def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_order_open, fee, +def test_trailing_stop_loss_positive(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, caplog, mocker) -> None: - buy_price = limit_buy_order['price'] + buy_price = limit_buy_order_usdt['price'] patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -3286,7 +3289,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_or 'last': buy_price - 0.000001 }), create_order=MagicMock(side_effect=[ - limit_buy_order_open, + limit_buy_order_usdt_open, {'id': 1234553382}, ]), get_fee=fee, @@ -3301,7 +3304,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_or freqtrade.enter_positions() trade = Trade.query.first() - trade.update(limit_buy_order) + trade.update(limit_buy_order_usdt) caplog.set_level(logging.DEBUG) # stop-loss not reached assert freqtrade.handle_trade(trade) is False @@ -3334,9 +3337,9 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_or f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog) -def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_order_open, fee, +def test_trailing_stop_loss_offset(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, caplog, mocker) -> None: - buy_price = limit_buy_order['price'] + buy_price = limit_buy_order_usdt['price'] patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -3347,7 +3350,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_orde 'last': buy_price - 0.000001 }), create_order=MagicMock(side_effect=[ - limit_buy_order_open, + limit_buy_order_usdt_open, {'id': 1234553382}, ]), get_fee=fee, @@ -3362,7 +3365,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_orde freqtrade.enter_positions() trade = Trade.query.first() - trade.update(limit_buy_order) + trade.update(limit_buy_order_usdt) caplog.set_level(logging.DEBUG) # stop-loss not reached assert freqtrade.handle_trade(trade) is False @@ -3396,10 +3399,10 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_orde assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value -def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_open, fee, +def test_tsl_only_offset_reached(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, caplog, mocker) -> None: - buy_price = limit_buy_order['price'] - # buy_price: 0.00001099 + buy_price = limit_buy_order_usdt['price'] + # buy_price: 2.0 patch_RPCManager(mocker) patch_exchange(mocker) @@ -3410,7 +3413,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_ 'ask': buy_price, 'last': buy_price }), - create_order=MagicMock(return_value=limit_buy_order_open), + create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) patch_whitelist(mocker, default_conf) @@ -3425,11 +3428,11 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_ freqtrade.enter_positions() trade = Trade.query.first() - trade.update(limit_buy_order) + trade.update(limit_buy_order_usdt) caplog.set_level(logging.DEBUG) # stop-loss not reached assert freqtrade.handle_trade(trade) is False - assert trade.stop_loss == 0.0000098910 + assert trade.stop_loss == 1.8 # Raise ticker above buy price mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', @@ -3443,7 +3446,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_ assert freqtrade.handle_trade(trade) is False assert not log_has("ETH/BTC - Adjusting stoploss...", caplog) - assert trade.stop_loss == 0.0000098910 + assert trade.stop_loss == 1.8 caplog.clear() # price rises above the offset (rises 12% when the offset is 5.5%) @@ -3460,7 +3463,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_ assert trade.stop_loss == 0.0000117705 -def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, +def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -3472,7 +3475,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_b 'last': 0.00000172 }), create_order=MagicMock(side_effect=[ - limit_buy_order_open, + limit_buy_order_usdt_open, {'id': 1234553382}, {'id': 1234553383} ]), @@ -3489,7 +3492,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_b freqtrade.enter_positions() trade = Trade.query.first() - trade.update(limit_buy_order) + trade.update(limit_buy_order_usdt) # Sell due to min_roi_reached patch_get_signal(freqtrade, value=(True, True, None)) assert freqtrade.handle_trade(trade) is True @@ -3678,8 +3681,8 @@ def test_get_real_amount_multi( def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, fee, mocker): - limit_buy_order = deepcopy(buy_order_fee) - limit_buy_order['fee'] = {'cost': 0.004} + limit_buy_order_usdt = deepcopy(buy_order_fee) + limit_buy_order_usdt['fee'] = {'cost': 0.004} mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) amount = float(sum(x['amount'] for x in trades_for_order)) @@ -3695,12 +3698,12 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change - assert freqtrade.get_real_amount(trade, limit_buy_order) == amount + assert freqtrade.get_real_amount(trade, limit_buy_order_usdt) == amount def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_fee, fee, mocker): - limit_buy_order = deepcopy(buy_order_fee) - limit_buy_order['amount'] = limit_buy_order['amount'] - 0.001 + limit_buy_order_usdt = deepcopy(buy_order_fee) + limit_buy_order_usdt['amount'] = limit_buy_order_usdt['amount'] - 0.001 mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = float(sum(x['amount'] for x in trades_for_order)) @@ -3717,13 +3720,13 @@ def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_ # Amount does not change with pytest.raises(DependencyException, match=r"Half bought\? Amounts don't match"): - freqtrade.get_real_amount(trade, limit_buy_order) + freqtrade.get_real_amount(trade, limit_buy_order_usdt) def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, buy_order_fee, fee, mocker): # Floats should not be compared directly. - limit_buy_order = deepcopy(buy_order_fee) + limit_buy_order_usdt = deepcopy(buy_order_fee) trades_for_order[0]['amount'] = trades_for_order[0]['amount'] + 1e-15 mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) @@ -3740,7 +3743,7 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount changes by fee amount. - assert isclose(freqtrade.get_real_amount(trade, limit_buy_order), amount - (amount * 0.001), + assert isclose(freqtrade.get_real_amount(trade, limit_buy_order_usdt), amount - (amount * 0.001), abs_tol=MATH_CLOSE_PREC,) @@ -3798,7 +3801,7 @@ def test_apply_fee_conditional(default_conf, fee, caplog, mocker, (0.1, False), (100, True), ]) -def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open, limit_buy_order, +def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_usdt_open, limit_buy_order_usdt, fee, mocker, order_book_l2, delta, is_high_delta): default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = delta @@ -3808,7 +3811,7 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - create_order=MagicMock(return_value=limit_buy_order_open), + create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) @@ -3831,9 +3834,9 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open, assert len(Trade.query.all()) == 1 # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + trade.update(limit_buy_order_usdt) - assert trade.open_rate == 0.00001099 + assert trade.open_rate == 2.0 assert whitelist == default_conf['exchange']['pair_whitelist'] @@ -3890,8 +3893,8 @@ def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: assert freqtrade._check_depth_of_market_buy('ETH/BTC', conf) is False -def test_order_book_ask_strategy(default_conf, limit_buy_order_open, limit_buy_order, fee, - limit_sell_order_open, mocker, order_book_l2, caplog) -> None: +def test_order_book_ask_strategy(default_conf, limit_buy_order_usdt_open, limit_buy_order_usdt, fee, + limit_sell_order_usdt_open, mocker, order_book_l2, caplog) -> None: """ test order book ask strategy """ @@ -3905,13 +3908,13 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order_open, limit_buy_o mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 + 'bid': 1.9, + 'ask': 2.2, + 'last': 1.9 }), create_order=MagicMock(side_effect=[ - limit_buy_order_open, - limit_sell_order_open, + limit_buy_order_usdt_open, + limit_sell_order_usdt_open, ]), get_fee=fee, ) @@ -3924,7 +3927,7 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order_open, limit_buy_o assert trade time.sleep(0.01) # Race condition fix - trade.update(limit_buy_order) + trade.update(limit_buy_order_usdt) freqtrade.wallets.update() assert trade.is_open is True @@ -3967,7 +3970,7 @@ def test_startup_trade_reinit(default_conf, edge_conf, mocker): @pytest.mark.usefixtures("init_persistence") -def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order_open, caplog): +def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order_usdt_open, caplog): default_conf['dry_run'] = True # Initialize to 2 times stake amount default_conf['dry_run_wallet'] = 0.002 @@ -3977,7 +3980,7 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order_ mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - create_order=MagicMock(return_value=limit_buy_order_open), + create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) @@ -3999,11 +4002,11 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order_ @pytest.mark.usefixtures("init_persistence") -def test_cancel_all_open_orders(mocker, default_conf, fee, limit_buy_order, limit_sell_order): +def test_cancel_all_open_orders(mocker, default_conf, fee, limit_buy_order_usdt, limit_sell_order_usdt): default_conf['cancel_open_orders_on_exit'] = True mocker.patch('freqtrade.exchange.Exchange.fetch_order', side_effect=[ - ExchangeError(), limit_sell_order, limit_buy_order, limit_sell_order]) + ExchangeError(), limit_sell_order_usdt, limit_buy_order_usdt, limit_sell_order_usdt]) buy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_enter') sell_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_exit') From 8d7f75c4de748f9d0784dc20b276b97d19e4a979 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 17 Sep 2021 22:18:14 -0600 Subject: [PATCH 163/239] Fixed a bunch of freqtradebot tests --- tests/conftest.py | 28 +--- tests/test_freqtradebot.py | 266 +++++++++++++------------------------ 2 files changed, 94 insertions(+), 200 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index c1032a215..adb344ffc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -338,8 +338,7 @@ def get_default_conf(testdatadir): "ETH/BTC", "LTC/BTC", "XRP/BTC", - "NEO/BTC", - "ADA/USDT" + "NEO/BTC" ], "pair_blacklist": [ "DOGE/BTC", @@ -469,31 +468,6 @@ def get_markets(): }, 'info': {}, }, - 'ADA/USDT': { - 'id': 'ethbtc', - 'symbol': 'ADA/USDT', - 'base': 'USDT', - 'quote': 'ADA', - 'active': True, - 'precision': { - 'price': 8, - 'amount': 8, - 'cost': 8, - }, - 'lot': 0.00000001, - 'limits': { - 'amount': { - 'min': 0.01, - 'max': 1000, - }, - 'price': 500000, - 'cost': { - 'min': 0.0001, - 'max': 500000, - }, - }, - 'info': {}, - }, 'TKN/BTC': { 'id': 'tknbtc', 'symbol': 'TKN/BTC', diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 7ddb90657..c030aca0f 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1267,11 +1267,14 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, ) # price fell below stoploss, so dry-run sells trade. - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ - 'bid': 4.1712, - 'ask': 4.1921, - 'last': 4.1712 - })) + mocker.patch( + 'freqtrade.exchange.Exchange.fetch_ticker', + MagicMock(return_value={ + 'bid': 4.1712, + 'ask': 4.1921, + 'last': 4.1712 + }) + ) assert freqtrade.handle_trade(trade) is True @@ -1407,7 +1410,7 @@ def test_handle_stoploss_on_exchange_custom_stop(mocker, default_conf, fee, 'price': 3, 'average': 2, 'info': { - 'stopPrice': '0.000011134' + 'stopPrice': '2.0805' } }) @@ -1417,11 +1420,14 @@ def test_handle_stoploss_on_exchange_custom_stop(mocker, default_conf, fee, assert freqtrade.handle_stoploss_on_exchange(trade) is False # price jumped 2x - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ - 'bid': 4.38, - 'ask': 4.4, - 'last': 4.38 - })) + mocker.patch( + 'freqtrade.exchange.Exchange.fetch_ticker', + MagicMock(return_value={ + 'bid': 4.38, + 'ask': 4.4, + 'last': 4.38 + }) + ) cancel_order_mock = MagicMock() stoploss_order_mock = MagicMock(return_value={'id': 13434334}) @@ -1435,7 +1441,7 @@ def test_handle_stoploss_on_exchange_custom_stop(mocker, default_conf, fee, stoploss_order_mock.assert_not_called() assert freqtrade.handle_trade(trade) is False - assert trade.stop_loss == 0.00002346 * 0.96 + assert trade.stop_loss == 4.4 * 0.96 assert trade.stop_loss_pct == -0.04 # setting stoploss_on_exchange_interval to 0 seconds @@ -1444,17 +1450,22 @@ def test_handle_stoploss_on_exchange_custom_stop(mocker, default_conf, fee, assert freqtrade.handle_stoploss_on_exchange(trade) is False cancel_order_mock.assert_called_once_with(100, 'ETH/BTC') - stoploss_order_mock.assert_called_once_with(amount=85.32423208, - pair='ETH/BTC', - order_types=freqtrade.strategy.order_types, - stop_price=0.00002346 * 0.96) + stoploss_order_mock.assert_called_once_with( + amount=100.0, + pair='ETH/BTC', + order_types=freqtrade.strategy.order_types, + stop_price=4.4 * 0.96 + ) # price fell below stoploss, so dry-run sells trade. - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ - 'bid': 0.00002144, - 'ask': 0.00002146, - 'last': 0.00002144 - })) + mocker.patch( + 'freqtrade.exchange.Exchange.fetch_ticker', + MagicMock(return_value={ + 'bid': 4.1712, + 'ask': 4.1921, + 'last': 4.1712 + }) + ) assert freqtrade.handle_trade(trade) is True @@ -3192,9 +3203,9 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order_usdt, limit_buy_ mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.0000172, - 'ask': 0.0000173, - 'last': 0.0000172 + 'bid': 2.19, + 'ask': 2.2, + 'last': 2.19 }), create_order=MagicMock(side_effect=[ limit_buy_order_usdt_open, @@ -3259,7 +3270,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order_usdt_open, limit_buy_o # Stoploss should be adjusted assert freqtrade.handle_trade(trade) is False - + caplog.clear() # Price fell mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ @@ -3271,22 +3282,28 @@ def test_trailing_stop_loss(default_conf, limit_buy_order_usdt_open, limit_buy_o caplog.set_level(logging.DEBUG) # Sell as trailing-stop is reached assert freqtrade.handle_trade(trade) is True - assert log_has("ETH/BTC - HIT STOP: current price at 0.000012, stoploss is 0.000015, " - "initial stoploss was at 0.000010, trade opened at 0.000011", caplog) + # TODO: Does this make sense? How is stoploss 2.7? + assert log_has("ETH/BTC - HIT STOP: current price at 2.200000, stoploss is 2.700000, " + "initial stoploss was at 1.800000, trade opened at 2.000000", caplog) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value -def test_trailing_stop_loss_positive(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, - caplog, mocker) -> None: +@pytest.mark.parametrize('offset,trail_if_reached,second_sl', [ + (0, False, 2.0394), + (0.011, False, 2.0394), + (0.055, True, 1.8), +]) +def test_trailing_stop_loss_positive(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, + offset, fee, caplog, mocker, trail_if_reached, second_sl) -> None: buy_price = limit_buy_order_usdt['price'] patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': buy_price - 0.000001, - 'ask': buy_price - 0.000001, - 'last': buy_price - 0.000001 + 'bid': buy_price - 0.01, + 'ask': buy_price - 0.01, + 'last': buy_price - 0.01 }), create_order=MagicMock(side_effect=[ limit_buy_order_usdt_open, @@ -3296,6 +3313,9 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order_usdt, limit_b ) default_conf['trailing_stop'] = True default_conf['trailing_stop_positive'] = 0.01 + if offset: + default_conf['trailing_stop_positive_offset'] = offset + default_conf['trailing_only_offset_is_reached'] = trail_if_reached patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -3310,159 +3330,59 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order_usdt, limit_b assert freqtrade.handle_trade(trade) is False # Raise ticker above buy price - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - MagicMock(return_value={ - 'bid': buy_price + 0.000003, - 'ask': buy_price + 0.000003, - 'last': buy_price + 0.000003 - })) - # stop-loss not reached, adjusted stoploss - assert freqtrade.handle_trade(trade) is False - assert log_has("ETH/BTC - Using positive stoploss: 0.01 offset: 0 profit: 0.2666%", caplog) - assert log_has("ETH/BTC - Adjusting stoploss...", caplog) - assert trade.stop_loss == 0.0000138501 - caplog.clear() - - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - MagicMock(return_value={ - 'bid': buy_price + 0.000002, - 'ask': buy_price + 0.000002, - 'last': buy_price + 0.000002 - })) - # Lower price again (but still positive) - assert freqtrade.handle_trade(trade) is True - assert log_has( - f"ETH/BTC - HIT STOP: current price at {buy_price + 0.000002:.6f}, " - f"stoploss is {trade.stop_loss:.6f}, " - f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog) - - -def test_trailing_stop_loss_offset(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, - caplog, mocker) -> None: - buy_price = limit_buy_order_usdt['price'] - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=MagicMock(return_value={ - 'bid': buy_price - 0.000001, - 'ask': buy_price - 0.000001, - 'last': buy_price - 0.000001 - }), - create_order=MagicMock(side_effect=[ - limit_buy_order_usdt_open, - {'id': 1234553382}, - ]), - get_fee=fee, + mocker.patch( + 'freqtrade.exchange.Exchange.fetch_ticker', + MagicMock(return_value={ + 'bid': buy_price + 0.06, + 'ask': buy_price + 0.06, + 'last': buy_price + 0.06 + }) ) - patch_whitelist(mocker, default_conf) - default_conf['trailing_stop'] = True - default_conf['trailing_stop_positive'] = 0.01 - default_conf['trailing_stop_positive_offset'] = 0.011 - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.enter_positions() - - trade = Trade.query.first() - trade.update(limit_buy_order_usdt) - caplog.set_level(logging.DEBUG) - # stop-loss not reached - assert freqtrade.handle_trade(trade) is False - - # Raise ticker above buy price - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - MagicMock(return_value={ - 'bid': buy_price + 0.000003, - 'ask': buy_price + 0.000003, - 'last': buy_price + 0.000003 - })) # stop-loss not reached, adjusted stoploss assert freqtrade.handle_trade(trade) is False - assert log_has("ETH/BTC - Using positive stoploss: 0.01 offset: 0.011 profit: 0.2666%", caplog) - assert log_has("ETH/BTC - Adjusting stoploss...", caplog) - assert trade.stop_loss == 0.0000138501 + # TODO: is 0.0249% correct? Shouldn't it be higher? + caplog_text = f"ETH/BTC - Using positive stoploss: 0.01 offset: {offset} profit: 0.0249%" + if trail_if_reached: + assert not log_has(caplog_text, caplog) + assert not log_has("ETH/BTC - Adjusting stoploss...", caplog) + else: + assert log_has(caplog_text, caplog) + assert log_has("ETH/BTC - Adjusting stoploss...", caplog) + assert trade.stop_loss == second_sl caplog.clear() - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - MagicMock(return_value={ - 'bid': buy_price + 0.000002, - 'ask': buy_price + 0.000002, - 'last': buy_price + 0.000002 - })) + mocker.patch( + 'freqtrade.exchange.Exchange.fetch_ticker', + MagicMock(return_value={ + 'bid': buy_price + 0.125, + 'ask': buy_price + 0.125, + 'last': buy_price + 0.125, + }) + ) + assert freqtrade.handle_trade(trade) is False + assert log_has( + f"ETH/BTC - Using positive stoploss: 0.01 offset: {offset} profit: 0.0572%", + caplog + ) + assert log_has("ETH/BTC - Adjusting stoploss...", caplog) + + mocker.patch( + 'freqtrade.exchange.Exchange.fetch_ticker', + MagicMock(return_value={ + 'bid': buy_price + 0.02, + 'ask': buy_price + 0.02, + 'last': buy_price + 0.02 + }) + ) # Lower price again (but still positive) assert freqtrade.handle_trade(trade) is True assert log_has( - f"ETH/BTC - HIT STOP: current price at {buy_price + 0.000002:.6f}, " + f"ETH/BTC - HIT STOP: current price at {buy_price + 0.02:.6f}, " f"stoploss is {trade.stop_loss:.6f}, " - f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog) + f"initial stoploss was at 1.800000, trade opened at 2.000000", caplog) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value -def test_tsl_only_offset_reached(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, - caplog, mocker) -> None: - buy_price = limit_buy_order_usdt['price'] - # buy_price: 2.0 - - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=MagicMock(return_value={ - 'bid': buy_price, - 'ask': buy_price, - 'last': buy_price - }), - create_order=MagicMock(return_value=limit_buy_order_usdt_open), - get_fee=fee, - ) - patch_whitelist(mocker, default_conf) - default_conf['trailing_stop'] = True - default_conf['trailing_stop_positive'] = 0.05 - default_conf['trailing_stop_positive_offset'] = 0.055 - default_conf['trailing_only_offset_is_reached'] = True - - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.enter_positions() - - trade = Trade.query.first() - trade.update(limit_buy_order_usdt) - caplog.set_level(logging.DEBUG) - # stop-loss not reached - assert freqtrade.handle_trade(trade) is False - assert trade.stop_loss == 1.8 - - # Raise ticker above buy price - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - MagicMock(return_value={ - 'bid': buy_price + 0.0000004, - 'ask': buy_price + 0.0000004, - 'last': buy_price + 0.0000004 - })) - - # stop-loss should not be adjusted as offset is not reached yet - assert freqtrade.handle_trade(trade) is False - - assert not log_has("ETH/BTC - Adjusting stoploss...", caplog) - assert trade.stop_loss == 1.8 - caplog.clear() - - # price rises above the offset (rises 12% when the offset is 5.5%) - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - MagicMock(return_value={ - 'bid': buy_price + 0.0000014, - 'ask': buy_price + 0.0000014, - 'last': buy_price + 0.0000014 - })) - - assert freqtrade.handle_trade(trade) is False - assert log_has("ETH/BTC - Using positive stoploss: 0.05 offset: 0.055 profit: 0.1218%", caplog) - assert log_has("ETH/BTC - Adjusting stoploss...", caplog) - assert trade.stop_loss == 0.0000117705 - - def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, mocker) -> None: patch_RPCManager(mocker) From d1e3d480751e53a240a3c137770fd18f60a67607 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 26 Sep 2021 02:43:47 -0600 Subject: [PATCH 164/239] changed test_update_trade_state_withorderdict to usdt --- tests/conftest.py | 49 ++++++++++++++++++++++---------------- tests/test_freqtradebot.py | 8 +++++-- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index adb344ffc..09725213f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1598,27 +1598,34 @@ def result(testdatadir): @pytest.fixture(scope="function") def trades_for_order(): - return [{'info': {'id': 34567, - 'orderId': 123456, - 'price': '0.24544100', - 'qty': '8.00000000', - 'commission': '0.00800000', - 'commissionAsset': 'LTC', - 'time': 1521663363189, - 'isBuyer': True, - 'isMaker': False, - 'isBestMatch': True}, - 'timestamp': 1521663363189, - 'datetime': '2018-03-21T20:16:03.189Z', - 'symbol': 'LTC/ETH', - 'id': '34567', - 'order': '123456', - 'type': None, - 'side': 'buy', - 'price': 0.245441, - 'cost': 1.963528, - 'amount': 8.0, - 'fee': {'cost': 0.008, 'currency': 'LTC'}}] + return [{ + 'info': { + 'id': 34567, + 'orderId': 123456, + 'price': '0.24544100', + 'qty': '8.00000000', + 'commission': '0.00800000', + 'commissionAsset': 'LTC', + 'time': 1521663363189, + 'isBuyer': True, + 'isMaker': False, + 'isBestMatch': True + }, + 'timestamp': 1521663363189, + 'datetime': '2018-03-21T20:16:03.189Z', + 'symbol': 'LTC/USDT', + 'id': '34567', + 'order': '123456', + 'type': None, + 'side': 'buy', + 'price': 2.0, + 'cost': 16.0, + 'amount': 8.0, + 'fee': { + 'cost': 0.008, + 'currency': 'LTC' + } + }] @pytest.fixture(scope="function") diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index c030aca0f..dc5688122 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1207,6 +1207,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, patch_get_signal(freqtrade) freqtrade.enter_positions() + # TODO-lev: Get this trade switched to the usdt trades trade = Trade.query.first() trade.is_open = True trade.open_order_id = None @@ -1709,17 +1710,20 @@ def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_ patch_exchange(mocker) amount = sum(x['amount'] for x in trades_for_order) freqtrade = get_patched_freqtradebot(mocker, default_conf) + caplog.clear() trade = Trade( - pair='LTC/ETH', + pair='LTC/USDT', amount=amount, exchange='binance', - open_rate=0.245441, + open_rate=2.0, open_date=arrow.utcnow().datetime, fee_open=fee.return_value, fee_close=fee.return_value, open_order_id="123456", is_open=True, ) + # TODO-lev: caplog.text has Amount 60.00000000000001 does not match amount 60.00000000000001 + # TODO-lev: but they are the exact same freqtrade.update_trade_state(trade, '123456', limit_buy_order_usdt) assert trade.amount != amount assert trade.amount == limit_buy_order_usdt['amount'] From 6fdcf8cd73151291910f3ac87af80c4d743fa0cd Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 26 Sep 2021 10:09:10 -0600 Subject: [PATCH 165/239] created default_conf_usdt and init_persistence_usdt so that these tests pass: test_handle_stoploss_on_exchange_trailing, test_handle_stoploss_on_exchange_custom_stop, test_update_trade_state_withorderdict --- tests/conftest.py | 33 ++++++++++++++++++++-- tests/test_freqtradebot.py | 56 ++++++++++++++++++-------------------- 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 09725213f..c4ed05254 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -289,11 +289,21 @@ def init_persistence(default_conf): init_db(default_conf['db_url'], default_conf['dry_run']) +@pytest.fixture(scope='function') +def init_persistence_usdt(default_conf_usdt): + init_db(default_conf_usdt['db_url'], default_conf_usdt['dry_run']) + + @pytest.fixture(scope="function") def default_conf(testdatadir): return get_default_conf(testdatadir) +@pytest.fixture(scope="function") +def default_conf_usdt(testdatadir): + return get_default_conf_usdt(testdatadir) + + def get_default_conf(testdatadir): """ Returns validated configuration suitable for most tests """ configuration = { @@ -368,6 +378,15 @@ def get_default_conf(testdatadir): return configuration +def get_default_conf_usdt(testdatadir): + configuration = get_default_conf(testdatadir) + configuration.update({ + "stake_amount": 60.0, + "stake_currency": "USDT", + }) + return configuration + + @pytest.fixture def update(): _update = Update(0) @@ -1602,7 +1621,7 @@ def trades_for_order(): 'info': { 'id': 34567, 'orderId': 123456, - 'price': '0.24544100', + 'price': '2.0', 'qty': '8.00000000', 'commission': '0.00800000', 'commissionAsset': 'LTC', @@ -1811,6 +1830,14 @@ def edge_conf(default_conf): return conf +@pytest.fixture(scope="function") +def edge_conf_usdt(edge_conf): + edge_conf.update({ + "stake_currency": "USDT", + }) + return edge_conf + + @pytest.fixture def rpc_balance(): return { @@ -2049,7 +2076,7 @@ def saved_hyperopt_results(): @pytest.fixture(scope='function') def limit_buy_order_usdt_open(): return { - 'id': 'mocked_limit_buy', + 'id': 'mocked_limit_buy_usdt', 'type': 'limit', 'side': 'buy', 'symbol': 'mocked', @@ -2076,7 +2103,7 @@ def limit_buy_order_usdt(limit_buy_order_usdt_open): @pytest.fixture def limit_sell_order_usdt_open(): return { - 'id': 'mocked_limit_sell', + 'id': 'mocked_limit_sell_usdt', 'type': 'limit', 'side': 'sell', 'pair': 'mocked', diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index dc5688122..24ed3d509 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1162,8 +1162,8 @@ def test_create_stoploss_order_insufficient_funds(mocker, default_conf, caplog, assert mock_insuf.call_count == 1 -@pytest.mark.usefixtures("init_persistence") -def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, +@pytest.mark.usefixtures("init_persistence_usdt") +def test_handle_stoploss_on_exchange_trailing(mocker, default_conf_usdt, fee, limit_buy_order_usdt, limit_sell_order_usdt) -> None: # When trailing stoploss is set stoploss = MagicMock(return_value={'id': 13434334}) @@ -1188,12 +1188,12 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, ) # enabling TSL - default_conf['trailing_stop'] = True + default_conf_usdt['trailing_stop'] = True # disabling ROI - default_conf['minimal_roi']['0'] = 999999999 + default_conf_usdt['minimal_roi']['0'] = 999999999 - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) # enabling stoploss on exchange freqtrade.strategy.order_types['stoploss_on_exchange'] = True @@ -1207,7 +1207,6 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, patch_get_signal(freqtrade) freqtrade.enter_positions() - # TODO-lev: Get this trade switched to the usdt trades trade = Trade.query.first() trade.is_open = True trade.open_order_id = None @@ -1261,7 +1260,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, cancel_order_mock.assert_called_once_with(100, 'ETH/BTC') stoploss_order_mock.assert_called_once_with( - amount=30.0, + amount=27.39726027, pair='ETH/BTC', order_types=freqtrade.strategy.order_types, stop_price=4.4 * 0.95 @@ -1271,9 +1270,9 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, mocker.patch( 'freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ - 'bid': 4.1712, - 'ask': 4.1921, - 'last': 4.1712 + 'bid': 4.16, + 'ask': 4.17, + 'last': 4.16 }) ) assert freqtrade.handle_trade(trade) is True @@ -1354,9 +1353,9 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c assert log_has_re(r"Could not create trailing stoploss order for pair ETH/BTC\..*", caplog) -@pytest.mark.usefixtures("init_persistence") -def test_handle_stoploss_on_exchange_custom_stop(mocker, default_conf, fee, - limit_buy_order_usdt, limit_sell_order_usdt) -> None: +@pytest.mark.usefixtures("init_persistence_usdt") +def test_handle_stoploss_on_exchange_custom_stop( + mocker, default_conf_usdt, fee, limit_buy_order_usdt, limit_sell_order_usdt) -> None: # When trailing stoploss is set stoploss = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) @@ -1380,12 +1379,12 @@ def test_handle_stoploss_on_exchange_custom_stop(mocker, default_conf, fee, ) # enabling TSL - default_conf['use_custom_stoploss'] = True + default_conf_usdt['use_custom_stoploss'] = True # disabling ROI - default_conf['minimal_roi']['0'] = 999999999 + default_conf_usdt['minimal_roi']['0'] = 999999999 - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) # enabling stoploss on exchange freqtrade.strategy.order_types['stoploss_on_exchange'] = True @@ -1452,7 +1451,7 @@ def test_handle_stoploss_on_exchange_custom_stop(mocker, default_conf, fee, cancel_order_mock.assert_called_once_with(100, 'ETH/BTC') stoploss_order_mock.assert_called_once_with( - amount=100.0, + amount=31.57894736, pair='ETH/BTC', order_types=freqtrade.strategy.order_types, stop_price=4.4 * 0.96 @@ -1462,9 +1461,9 @@ def test_handle_stoploss_on_exchange_custom_stop(mocker, default_conf, fee, mocker.patch( 'freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ - 'bid': 4.1712, - 'ask': 4.1921, - 'last': 4.1712 + 'bid': 4.17, + 'ask': 4.19, + 'last': 4.17 }) ) assert freqtrade.handle_trade(trade) is True @@ -1554,7 +1553,6 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, 'ask': 2.2 * 0.95, 'last': 1.9 * 0.95 })) - assert freqtrade.handle_trade(trade) is False assert freqtrade.handle_stoploss_on_exchange(trade) is False @@ -1566,21 +1564,21 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, # price jumped 2x mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ - 'bid': 0.00002344, - 'ask': 0.00002346, - 'last': 0.00002344 + 'bid': 4.38, + 'ask': 4.4, + 'last': 4.38 })) assert freqtrade.handle_trade(trade) is False assert freqtrade.handle_stoploss_on_exchange(trade) is False # stoploss should be set to 1% as trailing is on - assert trade.stop_loss == 0.00002346 * 0.99 + assert trade.stop_loss == 4.4 * 0.99 cancel_order_mock.assert_called_once_with(100, 'NEO/BTC') stoploss_order_mock.assert_called_once_with(amount=2132892.49146757, pair='NEO/BTC', order_types=freqtrade.strategy.order_types, - stop_price=0.00002346 * 0.99) + stop_price=4.4 * 0.99) @pytest.mark.parametrize('return_value,side_effect,log_message', [ @@ -1701,7 +1699,7 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order_usdt, caplog) (30.0 + 1e-14, True), (8.0, False) ]) -def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_buy_order_usdt, fee, +def test_update_trade_state_withorderdict(default_conf_usdt, trades_for_order, limit_buy_order_usdt, fee, mocker, initial_amount, has_rounding_fee, caplog): trades_for_order[0]['amount'] = initial_amount mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) @@ -1709,7 +1707,7 @@ def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_ mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError)) patch_exchange(mocker) amount = sum(x['amount'] for x in trades_for_order) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) caplog.clear() trade = Trade( pair='LTC/USDT', @@ -1722,8 +1720,6 @@ def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_ open_order_id="123456", is_open=True, ) - # TODO-lev: caplog.text has Amount 60.00000000000001 does not match amount 60.00000000000001 - # TODO-lev: but they are the exact same freqtrade.update_trade_state(trade, '123456', limit_buy_order_usdt) assert trade.amount != amount assert trade.amount == limit_buy_order_usdt['amount'] From ffa9a3ac7d057a56c1b47ec0380bfd5a230011ec Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 26 Sep 2021 10:18:22 -0600 Subject: [PATCH 166/239] changed default_conf_usdt stake_amount to 10 --- tests/conftest.py | 2 +- tests/test_freqtradebot.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index c4ed05254..13628a66d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -381,7 +381,7 @@ def get_default_conf(testdatadir): def get_default_conf_usdt(testdatadir): configuration = get_default_conf(testdatadir) configuration.update({ - "stake_amount": 60.0, + "stake_amount": 10.0, "stake_currency": "USDT", }) return configuration diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 24ed3d509..282fd68ea 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1260,7 +1260,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf_usdt, fee, cancel_order_mock.assert_called_once_with(100, 'ETH/BTC') stoploss_order_mock.assert_called_once_with( - amount=27.39726027, + amount=4.56621004, pair='ETH/BTC', order_types=freqtrade.strategy.order_types, stop_price=4.4 * 0.95 @@ -1451,7 +1451,7 @@ def test_handle_stoploss_on_exchange_custom_stop( cancel_order_mock.assert_called_once_with(100, 'ETH/BTC') stoploss_order_mock.assert_called_once_with( - amount=31.57894736, + amount=5.26315789, pair='ETH/BTC', order_types=freqtrade.strategy.order_types, stop_price=4.4 * 0.96 From 5ce09c751935f5436732c8e05dd209a42bd95976 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 26 Sep 2021 10:25:26 -0600 Subject: [PATCH 167/239] updated test_reupdate_enter_order_fees to usdt --- tests/test_freqtradebot.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 282fd68ea..2d0e0a776 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -4043,9 +4043,9 @@ def test_update_closed_trades_without_assigned_fees(mocker, default_conf, fee): assert trade.fee_close_currency is not None -@pytest.mark.usefixtures("init_persistence") -def test_reupdate_enter_order_fees(mocker, default_conf, fee, caplog): - freqtrade = get_patched_freqtradebot(mocker, default_conf) +@pytest.mark.usefixtures("init_persistence_usdt") +def test_reupdate_enter_order_fees(mocker, default_conf_usdt, fee, caplog): + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mock_uts = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.update_trade_state') create_mock_trades(fee) @@ -4063,13 +4063,13 @@ def test_reupdate_enter_order_fees(mocker, default_conf, fee, caplog): # Test with trade without orders trade = Trade( pair='XRP/ETH', - stake_amount=0.001, + stake_amount=60.0, fee_open=fee.return_value, fee_close=fee.return_value, open_date=arrow.utcnow().datetime, is_open=True, - amount=20, - open_rate=0.01, + amount=30, + open_rate=2.0, exchange='binance', ) Trade.query.session.add(trade) From d0e0d0ee01cc0d4f008f598c2bdd477a15b40d9d Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 26 Sep 2021 11:56:44 -0600 Subject: [PATCH 168/239] Removed init_persistence_usdt --- tests/conftest.py | 5 ----- tests/test_freqtradebot.py | 6 +++--- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 13628a66d..3e88c88aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -289,11 +289,6 @@ def init_persistence(default_conf): init_db(default_conf['db_url'], default_conf['dry_run']) -@pytest.fixture(scope='function') -def init_persistence_usdt(default_conf_usdt): - init_db(default_conf_usdt['db_url'], default_conf_usdt['dry_run']) - - @pytest.fixture(scope="function") def default_conf(testdatadir): return get_default_conf(testdatadir) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 2d0e0a776..a000829dd 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1162,7 +1162,7 @@ def test_create_stoploss_order_insufficient_funds(mocker, default_conf, caplog, assert mock_insuf.call_count == 1 -@pytest.mark.usefixtures("init_persistence_usdt") +@pytest.mark.usefixtures("init_persistence") def test_handle_stoploss_on_exchange_trailing(mocker, default_conf_usdt, fee, limit_buy_order_usdt, limit_sell_order_usdt) -> None: # When trailing stoploss is set @@ -1353,7 +1353,7 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c assert log_has_re(r"Could not create trailing stoploss order for pair ETH/BTC\..*", caplog) -@pytest.mark.usefixtures("init_persistence_usdt") +@pytest.mark.usefixtures("init_persistence") def test_handle_stoploss_on_exchange_custom_stop( mocker, default_conf_usdt, fee, limit_buy_order_usdt, limit_sell_order_usdt) -> None: # When trailing stoploss is set @@ -4043,7 +4043,7 @@ def test_update_closed_trades_without_assigned_fees(mocker, default_conf, fee): assert trade.fee_close_currency is not None -@pytest.mark.usefixtures("init_persistence_usdt") +@pytest.mark.usefixtures("init_persistence") def test_reupdate_enter_order_fees(mocker, default_conf_usdt, fee, caplog): freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mock_uts = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.update_trade_state') From 26fdad84685de96cb7f3eb9b7986317f3b8b990a Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 26 Sep 2021 11:58:55 -0600 Subject: [PATCH 169/239] Removed edge_conf_usdt --- tests/conftest.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3e88c88aa..898dcfbfd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1825,14 +1825,6 @@ def edge_conf(default_conf): return conf -@pytest.fixture(scope="function") -def edge_conf_usdt(edge_conf): - edge_conf.update({ - "stake_currency": "USDT", - }) - return edge_conf - - @pytest.fixture def rpc_balance(): return { From 755cc9cda1d5d75a6487b19a0daf289c2a0902b7 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 26 Sep 2021 12:39:30 -0600 Subject: [PATCH 170/239] Updated test_check_available_stake_amount to use default_conf_usdt --- tests/test_freqtradebot.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index a000829dd..d8cc2214a 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -126,16 +126,16 @@ def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None: @pytest.mark.parametrize("amend_last,wallet,max_open,lsamr,expected", [ - (False, 0.002, 2, 0.5, [0.001, None]), - (True, 0.002, 2, 0.5, [0.001, 0.00098]), - (False, 0.003, 3, 0.5, [0.001, 0.001, None]), - (True, 0.003, 3, 0.5, [0.001, 0.001, 0.00097]), - (False, 0.0022, 3, 0.5, [0.001, 0.001, None]), - (True, 0.0022, 3, 0.5, [0.001, 0.001, 0.0]), - (True, 0.0027, 3, 0.5, [0.001, 0.001, 0.000673]), - (True, 0.0022, 3, 1, [0.001, 0.001, 0.0]), + (False, 20, 2, 0.5, [10, None]), + (True, 20, 2, 0.5, [10, 9.8]), + (False, 30, 3, 0.5, [10, 10, None]), + (True, 30, 3, 0.5, [10, 10, 9.7]), + (False, 22, 3, 0.5, [10, 10, None]), + (True, 22, 3, 0.5, [10, 10, 0.0]), + (True, 27, 3, 0.5, [10, 10, 6.73]), + (True, 22, 3, 1, [10, 10, 0.0]), ]) -def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_buy_order_usdt_open, +def test_check_available_stake_amount(default_conf_usdt, ticker, mocker, fee, limit_buy_order_usdt_open, amend_last, wallet, max_open, lsamr, expected) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -145,12 +145,12 @@ def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_b create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee ) - default_conf['dry_run_wallet'] = wallet + default_conf_usdt['dry_run_wallet'] = wallet - default_conf['amend_last_stake_amount'] = amend_last - default_conf['last_stake_amount_min_ratio'] = lsamr + default_conf_usdt['amend_last_stake_amount'] = amend_last + default_conf_usdt['last_stake_amount_min_ratio'] = lsamr - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) for i in range(0, max_open): From 7eebb6bb2d305ab24f08d1e640efcc88913152c0 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 26 Sep 2021 12:41:09 -0600 Subject: [PATCH 171/239] updated test_create_trade to use default_conf_usdt --- tests/test_freqtradebot.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index d8cc2214a..280d2ce97 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -270,7 +270,7 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, fee) -> None: assert Trade.total_open_trades_stakes() == 1.97502e-03 -def test_create_trade(default_conf, ticker, limit_buy_order_usdt, fee, mocker) -> None: +def test_create_trade(default_conf_usdt, ticker, limit_buy_order_usdt, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -281,14 +281,14 @@ def test_create_trade(default_conf, ticker, limit_buy_order_usdt, fee, mocker) - ) # Save state of current whitelist - whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) - freqtrade = FreqtradeBot(default_conf) + whitelist = deepcopy(default_conf_usdt['exchange']['pair_whitelist']) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) freqtrade.create_trade('ETH/BTC') trade = Trade.query.first() assert trade is not None - assert trade.stake_amount == 0.001 + assert trade.stake_amount == 10.0 assert trade.is_open assert trade.open_date is not None assert trade.exchange == 'binance' @@ -299,7 +299,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order_usdt, fee, mocker) - assert trade.open_rate == 2.0 assert trade.amount == 30.0 - assert whitelist == default_conf['exchange']['pair_whitelist'] + assert whitelist == default_conf_usdt['exchange']['pair_whitelist'] def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order_usdt, From ba5d78f005210d51416387e65fa30c7a38623395 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 26 Sep 2021 13:04:58 -0600 Subject: [PATCH 172/239] swapped default_conf for default_conf_usdt and ticker for ticker_usdt --- tests/conftest.py | 22 +- tests/test_freqtradebot.py | 788 ++++++++++++++++++------------------- 2 files changed, 413 insertions(+), 397 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 898dcfbfd..f2da68dd2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -378,6 +378,22 @@ def get_default_conf_usdt(testdatadir): configuration.update({ "stake_amount": 10.0, "stake_currency": "USDT", + "exchange": { + "name": "binance", + "enabled": True, + "key": "key", + "secret": "secret", + "pair_whitelist": [ + "ETH/USDT", + "LTC/USDT", + "XRP/USDT", + "NEO/USDT" + ], + "pair_blacklist": [ + "DOGE/USDT", + "HOT/USDT", + ] + }, }) return configuration @@ -424,9 +440,9 @@ def ticker_sell_down(): @pytest.fixture def ticker_usdt(): return MagicMock(return_value={ - 'bid': 1.99, - 'ask': 2.0, - 'last': 1.99, + 'bid': 2.0, + 'ask': 2.01, + 'last': 2.0, }) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 280d2ce97..2a96783bc 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -41,33 +41,33 @@ def patch_RPCManager(mocker) -> MagicMock: # Unit tests -def test_freqtradebot_state(mocker, default_conf, markets) -> None: +def test_freqtradebot_state(mocker, default_conf_usdt, markets) -> None: mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) assert freqtrade.state is State.RUNNING - default_conf.pop('initial_state') - freqtrade = FreqtradeBot(default_conf) + default_conf_usdt.pop('initial_state') + freqtrade = FreqtradeBot(default_conf_usdt) assert freqtrade.state is State.STOPPED -def test_process_stopped(mocker, default_conf) -> None: +def test_process_stopped(mocker, default_conf_usdt) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) coo_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cancel_all_open_orders') freqtrade.process_stopped() assert coo_mock.call_count == 0 - default_conf['cancel_open_orders_on_exit'] = True - freqtrade = get_patched_freqtradebot(mocker, default_conf) + default_conf_usdt['cancel_open_orders_on_exit'] = True + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) freqtrade.process_stopped() assert coo_mock.call_count == 1 -def test_bot_cleanup(mocker, default_conf, caplog) -> None: +def test_bot_cleanup(mocker, default_conf_usdt, caplog) -> None: mock_cleanup = mocker.patch('freqtrade.freqtradebot.cleanup_db') coo_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cancel_all_open_orders') - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) freqtrade.cleanup() assert log_has('Cleaning up modules ...', caplog) assert mock_cleanup.call_count == 1 @@ -82,10 +82,10 @@ def test_bot_cleanup(mocker, default_conf, caplog) -> None: RunMode.DRY_RUN, RunMode.LIVE ]) -def test_order_dict(default_conf, mocker, runmode, caplog) -> None: +def test_order_dict(default_conf_usdt, mocker, runmode, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - conf = default_conf.copy() + conf = default_conf_usdt.copy() conf['runmode'] = runmode conf['order_types'] = { 'buy': 'market', @@ -102,7 +102,7 @@ def test_order_dict(default_conf, mocker, runmode, caplog) -> None: caplog.clear() # is left untouched - conf = default_conf.copy() + conf = default_conf_usdt.copy() conf['runmode'] = runmode conf['order_types'] = { 'buy': 'market', @@ -115,14 +115,14 @@ def test_order_dict(default_conf, mocker, runmode, caplog) -> None: assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) -def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None: +def test_get_trade_stake_amount(default_conf_usdt, ticker_usdt, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) - result = freqtrade.wallets.get_trade_stake_amount('ETH/BTC') - assert result == default_conf['stake_amount'] + result = freqtrade.wallets.get_trade_stake_amount('ETH/USDT') + assert result == default_conf_usdt['stake_amount'] @pytest.mark.parametrize("amend_last,wallet,max_open,lsamr,expected", [ @@ -135,13 +135,13 @@ def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None: (True, 27, 3, 0.5, [10, 10, 6.73]), (True, 22, 3, 1, [10, 10, 0.0]), ]) -def test_check_available_stake_amount(default_conf_usdt, ticker, mocker, fee, limit_buy_order_usdt_open, +def test_check_available_stake_amount(default_conf_usdt, ticker_usdt, mocker, fee, limit_buy_order_usdt_open, amend_last, wallet, max_open, lsamr, expected) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee ) @@ -156,12 +156,12 @@ def test_check_available_stake_amount(default_conf_usdt, ticker, mocker, fee, li if expected[i] is not None: limit_buy_order_usdt_open['id'] = str(i) - result = freqtrade.wallets.get_trade_stake_amount('ETH/BTC') + result = freqtrade.wallets.get_trade_stake_amount('ETH/USDT') assert pytest.approx(result) == expected[i] - freqtrade.execute_entry('ETH/BTC', result) + freqtrade.execute_entry('ETH/USDT', result) else: with pytest.raises(DependencyException): - freqtrade.wallets.get_trade_stake_amount('ETH/BTC') + freqtrade.wallets.get_trade_stake_amount('ETH/USDT') def test_edge_called_in_process(mocker, edge_conf) -> None: @@ -169,7 +169,7 @@ def test_edge_called_in_process(mocker, edge_conf) -> None: patch_edge(mocker) def _refresh_whitelist(list): - return ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'] + return ['ETH/USDT', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'] patch_exchange(mocker) freqtrade = FreqtradeBot(edge_conf) @@ -208,7 +208,7 @@ def test_edge_overrides_stoploss(limit_buy_order_usdt, fee, caplog, mocker, # Strategy stoploss is -0.1 but Edge imposes a stoploss at -0.2 # Thus, if price falls 21%, stoploss should be triggered # - # mocking the ticker: price is falling ... + # mocking the ticker_usdt: price is falling ... buy_price = limit_buy_order_usdt['price'] mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -238,24 +238,24 @@ def test_edge_overrides_stoploss(limit_buy_order_usdt, fee, caplog, mocker, assert trade.sell_reason == SellType.STOP_LOSS.value -def test_total_open_trades_stakes(mocker, default_conf, ticker, fee) -> None: +def test_total_open_trades_stakes(mocker, default_conf_usdt, ticker_usdt, fee) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - default_conf['stake_amount'] = 0.00098751 - default_conf['max_open_trades'] = 2 + default_conf_usdt['stake_amount'] = 10.0 + default_conf_usdt['max_open_trades'] = 2 mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, get_fee=fee, _is_dry_limit_order_filled=MagicMock(return_value=False), ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) freqtrade.enter_positions() trade = Trade.query.first() assert trade is not None - assert trade.stake_amount == 0.00098751 + assert trade.stake_amount == 10.0 assert trade.is_open assert trade.open_date is not None @@ -263,19 +263,19 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, fee) -> None: trade = Trade.query.order_by(Trade.id.desc()).first() assert trade is not None - assert trade.stake_amount == 0.00098751 + assert trade.stake_amount == 10.0 assert trade.is_open assert trade.open_date is not None - assert Trade.total_open_trades_stakes() == 1.97502e-03 + assert Trade.total_open_trades_stakes() == 20.0 -def test_create_trade(default_conf_usdt, ticker, limit_buy_order_usdt, fee, mocker) -> None: +def test_create_trade(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, get_fee=fee, _is_dry_limit_order_filled=MagicMock(return_value=False), ) @@ -284,7 +284,7 @@ def test_create_trade(default_conf_usdt, ticker, limit_buy_order_usdt, fee, mock whitelist = deepcopy(default_conf_usdt['exchange']['pair_whitelist']) freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) - freqtrade.create_trade('ETH/BTC') + freqtrade.create_trade('ETH/USDT') trade = Trade.query.first() assert trade is not None @@ -302,21 +302,21 @@ def test_create_trade(default_conf_usdt, ticker, limit_buy_order_usdt, fee, mock assert whitelist == default_conf_usdt['exchange']['pair_whitelist'] -def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order_usdt, +def test_create_trade_no_stake_amount(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - patch_wallet(mocker, free=default_conf['stake_amount'] * 0.5) + patch_wallet(mocker, free=default_conf_usdt['stake_amount'] * 0.5) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) with pytest.raises(DependencyException, match=r'.*stake amount.*'): - freqtrade.create_trade('ETH/BTC') + freqtrade.create_trade('ETH/USDT') @pytest.mark.parametrize('stake_amount,create,amount_enough,max_open_trades', [ @@ -326,7 +326,7 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order_usdt (UNLIMITED_STAKE_AMOUNT, False, True, 0), ]) def test_create_trade_minimal_amount( - default_conf, ticker, limit_buy_order_usdt_open, fee, mocker, + default_conf_usdt, ticker_usdt, limit_buy_order_usdt_open, fee, mocker, stake_amount, create, amount_enough, max_open_trades, caplog ) -> None: patch_RPCManager(mocker) @@ -334,47 +334,47 @@ def test_create_trade_minimal_amount( buy_mock = MagicMock(return_value=limit_buy_order_usdt_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=buy_mock, get_fee=fee, ) - default_conf['max_open_trades'] = max_open_trades - freqtrade = FreqtradeBot(default_conf) + default_conf_usdt['max_open_trades'] = max_open_trades + freqtrade = FreqtradeBot(default_conf_usdt) freqtrade.config['stake_amount'] = stake_amount patch_get_signal(freqtrade) if create: - assert freqtrade.create_trade('ETH/BTC') + assert freqtrade.create_trade('ETH/USDT') if amount_enough: rate, amount = buy_mock.call_args[1]['rate'], buy_mock.call_args[1]['amount'] - assert rate * amount <= default_conf['stake_amount'] + assert rate * amount <= default_conf_usdt['stake_amount'] else: assert log_has_re( r"Stake amount for pair .* is too small.*", caplog ) else: - assert not freqtrade.create_trade('ETH/BTC') + assert not freqtrade.create_trade('ETH/USDT') if not max_open_trades: - assert freqtrade.wallets.get_trade_stake_amount('ETH/BTC', freqtrade.edge) == 0 + assert freqtrade.wallets.get_trade_stake_amount('ETH/USDT', freqtrade.edge) == 0 @pytest.mark.parametrize('whitelist,positions', [ - (["ETH/BTC"], 1), # No pairs left + (["ETH/USDT"], 1), # No pairs left ([], 0), # No pairs in whitelist ]) -def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_usdt_open, fee, +def test_enter_positions_no_pairs_left(default_conf_usdt, ticker_usdt, limit_buy_order_usdt_open, fee, whitelist, positions, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) - default_conf['exchange']['pair_whitelist'] = whitelist - freqtrade = FreqtradeBot(default_conf) + default_conf_usdt['exchange']['pair_whitelist'] = whitelist + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) n = freqtrade.enter_positions() @@ -390,17 +390,17 @@ def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_usd @pytest.mark.usefixtures("init_persistence") -def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order_usdt, fee, +def test_enter_positions_global_pairlock(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(return_value={'id': limit_buy_order_usdt['id']}), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) n = freqtrade.enter_positions() message = r"Global pairlock active until.* Not creating new trades." @@ -416,8 +416,8 @@ def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order_u assert log_has_re(message, caplog) -def test_handle_protections(mocker, default_conf, fee): - default_conf['protections'] = [ +def test_handle_protections(mocker, default_conf_usdt, fee): + default_conf_usdt['protections'] = [ {"method": "CooldownPeriod", "stop_duration": 60}, { "method": "StoplossGuard", @@ -428,7 +428,7 @@ def test_handle_protections(mocker, default_conf, fee): } ] - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) freqtrade.protections._protection_handlers[1].global_stop = MagicMock( return_value=(True, arrow.utcnow().shift(hours=1).datetime, "asdf")) create_mock_trades(fee) @@ -439,8 +439,8 @@ def test_handle_protections(mocker, default_conf, fee): assert send_msg_mock.call_args_list[1][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER_GLOBAL -def test_create_trade_no_signal(default_conf, fee, mocker) -> None: - default_conf['dry_run'] = True +def test_create_trade_no_signal(default_conf_usdt, fee, mocker) -> None: + default_conf_usdt['dry_run'] = True patch_RPCManager(mocker) patch_exchange(mocker) @@ -448,32 +448,32 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: 'freqtrade.exchange.Exchange', get_fee=fee, ) - default_conf['stake_amount'] = 10 - freqtrade = FreqtradeBot(default_conf) + default_conf_usdt['stake_amount'] = 10 + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade, value=(False, False, None)) Trade.query = MagicMock() Trade.query.filter = MagicMock() - assert not freqtrade.create_trade('ETH/BTC') + assert not freqtrade.create_trade('ETH/USDT') @pytest.mark.parametrize("max_open", range(0, 5)) @pytest.mark.parametrize("tradable_balance_ratio,modifier", [(1.0, 1), (0.99, 0.8), (0.5, 0.5)]) -def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, limit_buy_order_usdt_open, +def test_create_trades_multiple_trades(default_conf_usdt, ticker_usdt, fee, mocker, limit_buy_order_usdt_open, max_open, tradable_balance_ratio, modifier) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - default_conf['max_open_trades'] = max_open - default_conf['tradable_balance_ratio'] = tradable_balance_ratio - default_conf['dry_run_wallet'] = 0.001 * max_open + default_conf_usdt['max_open_trades'] = max_open + default_conf_usdt['tradable_balance_ratio'] = tradable_balance_ratio + default_conf_usdt['dry_run_wallet'] = 10.0 * max_open mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) n = freqtrade.enter_positions() @@ -484,47 +484,47 @@ def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, limit_ assert len(trades) == max(int(max_open * modifier), 0) -def test_create_trades_preopen(default_conf, ticker, fee, mocker, limit_buy_order_usdt_open) -> None: +def test_create_trades_preopen(default_conf_usdt, ticker_usdt, fee, mocker, limit_buy_order_usdt_open) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - default_conf['max_open_trades'] = 4 + default_conf_usdt['max_open_trades'] = 4 mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) # Create 2 existing trades - freqtrade.execute_entry('ETH/BTC', default_conf['stake_amount']) - freqtrade.execute_entry('NEO/BTC', default_conf['stake_amount']) + freqtrade.execute_entry('ETH/USDT', default_conf_usdt['stake_amount']) + freqtrade.execute_entry('NEO/BTC', default_conf_usdt['stake_amount']) assert len(Trade.get_open_trades()) == 2 # Change order_id for new orders limit_buy_order_usdt_open['id'] = '123444' # Create 2 new trades using create_trades - assert freqtrade.create_trade('ETH/BTC') + assert freqtrade.create_trade('ETH/USDT') assert freqtrade.create_trade('NEO/BTC') trades = Trade.get_open_trades() assert len(trades) == 4 -def test_process_trade_creation(default_conf, ticker, limit_buy_order_usdt, limit_buy_order_usdt_open, +def test_process_trade_creation(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(return_value=limit_buy_order_usdt_open), fetch_order=MagicMock(return_value=limit_buy_order_usdt), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -536,45 +536,45 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order_usdt, limi assert len(trades) == 1 trade = trades[0] assert trade is not None - assert trade.stake_amount == default_conf['stake_amount'] + assert trade.stake_amount == default_conf_usdt['stake_amount'] assert trade.is_open assert trade.open_date is not None assert trade.exchange == 'binance' - assert trade.open_rate == 0.00001098 - assert trade.amount == 91.07468123 + assert trade.open_rate == 2.0 + assert trade.amount == 5.0 assert log_has( - 'Buy signal found: about create a new trade for ETH/BTC with stake_amount: 0.001 ...', + 'Buy signal found: about create a new trade for ETH/USDT with stake_amount: 10.0 ...', caplog ) -def test_process_exchange_failures(default_conf, ticker, mocker) -> None: +def test_process_exchange_failures(default_conf_usdt, ticker_usdt, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(side_effect=TemporaryError) ) sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) - worker = Worker(args=None, config=default_conf) + worker = Worker(args=None, config=default_conf_usdt) patch_get_signal(worker.freqtrade) worker._process_running() assert sleep_mock.has_calls() -def test_process_operational_exception(default_conf, ticker, mocker) -> None: +def test_process_operational_exception(default_conf_usdt, ticker_usdt, mocker) -> None: msg_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(side_effect=OperationalException) ) - worker = Worker(args=None, config=default_conf) + worker = Worker(args=None, config=default_conf_usdt) patch_get_signal(worker.freqtrade) assert worker.freqtrade.state == State.RUNNING @@ -584,17 +584,17 @@ def test_process_operational_exception(default_conf, ticker, mocker) -> None: assert 'OperationalException' in msg_mock.call_args_list[-1][0][0]['status'] -def test_process_trade_handling(default_conf, ticker, limit_buy_order_usdt_open, fee, mocker) -> None: +def test_process_trade_handling(default_conf_usdt, ticker_usdt, limit_buy_order_usdt_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(return_value=limit_buy_order_usdt_open), fetch_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -609,23 +609,23 @@ def test_process_trade_handling(default_conf, ticker, limit_buy_order_usdt_open, assert len(trades) == 1 -def test_process_trade_no_whitelist_pair(default_conf, ticker, limit_buy_order_usdt, +def test_process_trade_no_whitelist_pair(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, fee, mocker) -> None: """ Test process with trade not in pair list """ patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(return_value={'id': limit_buy_order_usdt['id']}), fetch_order=MagicMock(return_value=limit_buy_order_usdt), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) pair = 'BLK/BTC' # Ensure the pair is not in the whitelist! - assert pair not in default_conf['exchange']['pair_whitelist'] + assert pair not in default_conf_usdt['exchange']['pair_whitelist'] # create open trade not in whitelist Trade.query.session.add(Trade( @@ -639,7 +639,7 @@ def test_process_trade_no_whitelist_pair(default_conf, ticker, limit_buy_order_u exchange='binance', )) Trade.query.session.add(Trade( - pair='ETH/BTC', + pair='ETH/USDT', stake_amount=0.001, fee_open=fee.return_value, fee_close=fee.return_value, @@ -656,17 +656,17 @@ def test_process_trade_no_whitelist_pair(default_conf, ticker, limit_buy_order_u assert len(freqtrade.active_pair_whitelist) == len(set(freqtrade.active_pair_whitelist)) -def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: +def test_process_informative_pairs_added(default_conf_usdt, ticker_usdt, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) def _refresh_whitelist(list): - return ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'] + return ['ETH/USDT', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'] refresh_mock = MagicMock() mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(side_effect=TemporaryError), refresh_latest_ohlcv=refresh_mock, ) @@ -677,7 +677,7 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: ) mocker.patch('time.sleep', return_value=None) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) freqtrade.pairlists._validate_whitelist = _refresh_whitelist freqtrade.strategy.informative_pairs = inf_pairs # patch_get_signal(freqtrade) @@ -687,13 +687,13 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: assert refresh_mock.call_count == 1 assert ("BTC/ETH", "1m") in refresh_mock.call_args[0][0] assert ("ETH/USDT", "1h") in refresh_mock.call_args[0][0] - assert ("ETH/BTC", default_conf["timeframe"]) in refresh_mock.call_args[0][0] + assert ("ETH/USDT", default_conf_usdt["timeframe"]) in refresh_mock.call_args[0][0] -def test_execute_entry(mocker, default_conf, fee, limit_buy_order_usdt, limit_buy_order_usdt_open) -> None: +def test_execute_entry(mocker, default_conf_usdt, fee, limit_buy_order_usdt, limit_buy_order_usdt_open) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=False) stake_amount = 2 bid = 0.11 @@ -711,7 +711,7 @@ def test_execute_entry(mocker, default_conf, fee, limit_buy_order_usdt, limit_bu get_min_pair_stake_amount=MagicMock(return_value=1), get_fee=fee, ) - pair = 'ETH/BTC' + pair = 'ETH/USDT' assert not freqtrade.execute_entry(pair, stake_amount) assert buy_rate_mock.call_count == 1 @@ -853,8 +853,8 @@ def test_execute_entry(mocker, default_conf, fee, limit_buy_order_usdt, limit_bu assert trade.open_rate_requested == 10 -def test_execute_entry_confirm_error(mocker, default_conf, fee, limit_buy_order_usdt) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) +def test_execute_entry_confirm_error(mocker, default_conf_usdt, fee, limit_buy_order_usdt) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ @@ -868,7 +868,7 @@ def test_execute_entry_confirm_error(mocker, default_conf, fee, limit_buy_order_ get_fee=fee, ) stake_amount = 2 - pair = 'ETH/BTC' + pair = 'ETH/USDT' freqtrade.strategy.confirm_trade_entry = MagicMock(side_effect=ValueError) assert freqtrade.execute_entry(pair, stake_amount) @@ -885,7 +885,7 @@ def test_execute_entry_confirm_error(mocker, default_conf, fee, limit_buy_order_ assert not freqtrade.execute_entry(pair, stake_amount) -def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order_usdt) -> None: +def test_add_stoploss_on_exchange(mocker, default_conf_usdt, limit_buy_order_usdt) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) @@ -897,7 +897,7 @@ def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order_usdt) -> stoploss = MagicMock(return_value={'id': 13434334}) mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) freqtrade.strategy.order_types['stoploss_on_exchange'] = True trade = MagicMock() @@ -912,7 +912,7 @@ def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order_usdt) -> assert trade.is_open is True -def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, +def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, limit_buy_order_usdt, limit_sell_order_usdt) -> None: stoploss = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) @@ -934,7 +934,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, 'freqtrade.exchange.Binance', stoploss=stoploss ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) # First case: when stoploss is not yet set but the order is open @@ -1032,7 +1032,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, assert stoploss.call_count == 0 -def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, +def test_handle_sle_cancel_cant_recreate(mocker, default_conf_usdt, fee, caplog, limit_buy_order_usdt, limit_sell_order_usdt) -> None: # Sixth case: stoploss order was cancelled but couldn't create new one patch_RPCManager(mocker) @@ -1055,7 +1055,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, fetch_stoploss_order=MagicMock(return_value={'status': 'canceled', 'id': 100}), stoploss=MagicMock(side_effect=ExchangeError()), ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) freqtrade.enter_positions() @@ -1071,7 +1071,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, assert trade.is_open is True -def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, +def test_create_stoploss_order_invalid_order(mocker, default_conf_usdt, caplog, fee, limit_buy_order_usdt_open, limit_sell_order_usdt): rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) @@ -1094,7 +1094,7 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, fetch_order=MagicMock(return_value={'status': 'canceled'}), stoploss=MagicMock(side_effect=InvalidOrderException()), ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) freqtrade.strategy.order_types['stoploss_on_exchange'] = True @@ -1119,10 +1119,10 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, assert rpc_mock.call_args_list[1][0][0]['order_type'] == 'market' -def test_create_stoploss_order_insufficient_funds(mocker, default_conf, caplog, fee, +def test_create_stoploss_order_insufficient_funds(mocker, default_conf_usdt, caplog, fee, limit_buy_order_usdt_open, limit_sell_order_usdt): sell_mock = MagicMock(return_value={'id': limit_sell_order_usdt['id']}) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mock_insuf = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_insufficient_funds') mocker.patch.multiple( @@ -1258,10 +1258,10 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf_usdt, fee, assert freqtrade.handle_stoploss_on_exchange(trade) is False - cancel_order_mock.assert_called_once_with(100, 'ETH/BTC') + cancel_order_mock.assert_called_once_with(100, 'ETH/USDT') stoploss_order_mock.assert_called_once_with( amount=4.56621004, - pair='ETH/BTC', + pair='ETH/USDT', order_types=freqtrade.strategy.order_types, stop_price=4.4 * 0.95 ) @@ -1278,7 +1278,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf_usdt, fee, assert freqtrade.handle_trade(trade) is True -def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, caplog, +def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf_usdt, fee, caplog, limit_buy_order_usdt, limit_sell_order_usdt) -> None: # When trailing stoploss is set stoploss = MagicMock(return_value={'id': 13434334}) @@ -1304,9 +1304,9 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c ) # enabling TSL - default_conf['trailing_stop'] = True + default_conf_usdt['trailing_stop'] = True - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) # enabling stoploss on exchange freqtrade.strategy.order_types['stoploss_on_exchange'] = True @@ -1339,7 +1339,7 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', return_value=stoploss_order_hanging) freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging) - assert log_has_re(r"Could not cancel stoploss order abcd for pair ETH/BTC.*", caplog) + assert log_has_re(r"Could not cancel stoploss order abcd for pair ETH/USDT.*", caplog) # Still try to create order assert stoploss.call_count == 1 @@ -1350,7 +1350,7 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c mocker.patch("freqtrade.exchange.Binance.stoploss", side_effect=ExchangeError()) freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging) assert cancel_mock.call_count == 1 - assert log_has_re(r"Could not create trailing stoploss order for pair ETH/BTC\..*", caplog) + assert log_has_re(r"Could not create trailing stoploss order for pair ETH/USDT\..*", caplog) @pytest.mark.usefixtures("init_persistence") @@ -1449,10 +1449,10 @@ def test_handle_stoploss_on_exchange_custom_stop( assert freqtrade.handle_stoploss_on_exchange(trade) is False - cancel_order_mock.assert_called_once_with(100, 'ETH/BTC') + cancel_order_mock.assert_called_once_with(100, 'ETH/USDT') stoploss_order_mock.assert_called_once_with( amount=5.26315789, - pair='ETH/BTC', + pair='ETH/USDT', order_types=freqtrade.strategy.order_types, stop_price=4.4 * 0.96 ) @@ -1583,12 +1583,12 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, @pytest.mark.parametrize('return_value,side_effect,log_message', [ (False, None, 'Found no buy signals for whitelisted currencies. Trying again...'), - (None, DependencyException, 'Unable to create trade for ETH/BTC: ') + (None, DependencyException, 'Unable to create trade for ETH/USDT: ') ]) -def test_enter_positions(mocker, default_conf, return_value, side_effect, +def test_enter_positions(mocker, default_conf_usdt, return_value, side_effect, log_message, caplog) -> None: caplog.set_level(logging.DEBUG) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mock_ct = mocker.patch( 'freqtrade.freqtradebot.FreqtradeBot.create_trade', @@ -1601,11 +1601,11 @@ def test_enter_positions(mocker, default_conf, return_value, side_effect, assert n == 0 assert log_has(log_message, caplog) # create_trade should be called once for every pair in the whitelist. - assert mock_ct.call_count == len(default_conf['exchange']['pair_whitelist']) + assert mock_ct.call_count == len(default_conf_usdt['exchange']['pair_whitelist']) -def test_exit_positions(mocker, default_conf, limit_buy_order_usdt, caplog) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) +def test_exit_positions(mocker, default_conf_usdt, limit_buy_order_usdt, caplog) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order_usdt) @@ -1630,14 +1630,14 @@ def test_exit_positions(mocker, default_conf, limit_buy_order_usdt, caplog) -> N assert n == 0 -def test_exit_positions_exception(mocker, default_conf, limit_buy_order_usdt, caplog) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) +def test_exit_positions_exception(mocker, default_conf_usdt, limit_buy_order_usdt, caplog) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order_usdt) trade = MagicMock() trade.open_order_id = None trade.open_fee = 0.001 - trade.pair = 'ETH/BTC' + trade.pair = 'ETH/USDT' trades = [trade] # Test raise of DependencyException exception @@ -1647,11 +1647,11 @@ def test_exit_positions_exception(mocker, default_conf, limit_buy_order_usdt, ca ) n = freqtrade.exit_positions(trades) assert n == 0 - assert log_has('Unable to sell trade ETH/BTC: ', caplog) + assert log_has('Unable to sell trade ETH/USDT: ', caplog) -def test_update_trade_state(mocker, default_conf, limit_buy_order_usdt, caplog) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) +def test_update_trade_state(mocker, default_conf_usdt, limit_buy_order_usdt, caplog) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order_usdt) @@ -1727,9 +1727,9 @@ def test_update_trade_state_withorderdict(default_conf_usdt, trades_for_order, l assert log_has_re(r'Applying fee on amount for .*', caplog) -def test_update_trade_state_exception(mocker, default_conf, +def test_update_trade_state_exception(mocker, default_conf_usdt, limit_buy_order_usdt, caplog) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order_usdt) trade = MagicMock() @@ -1745,8 +1745,8 @@ def test_update_trade_state_exception(mocker, default_conf, assert log_has('Could not update trade amount: ', caplog) -def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) +def test_update_trade_state_orderexception(mocker, default_conf_usdt, caplog) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=InvalidOrderException)) @@ -1761,7 +1761,7 @@ def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None assert log_has(f'Unable to fetch order {trade.open_order_id}: ', caplog) -def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_order_usdt_open, +def test_update_trade_state_sell(default_conf_usdt, trades_for_order, limit_sell_order_usdt_open, limit_sell_order_usdt, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) # fetch_order should not be called!! @@ -1771,7 +1771,7 @@ def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_orde patch_exchange(mocker) amount = limit_sell_order_usdt["amount"] - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) wallet_mock.reset_mock() trade = Trade( pair='LTC/ETH', @@ -1796,7 +1796,7 @@ def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_orde assert order.status == 'closed' -def test_handle_trade(default_conf, limit_buy_order_usdt, limit_sell_order_usdt_open, limit_sell_order_usdt, +def test_handle_trade(default_conf_usdt, limit_buy_order_usdt, limit_sell_order_usdt_open, limit_sell_order_usdt, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -1813,7 +1813,7 @@ def test_handle_trade(default_conf, limit_buy_order_usdt, limit_sell_order_usdt_ ]), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) freqtrade.enter_positions() @@ -1839,13 +1839,13 @@ def test_handle_trade(default_conf, limit_buy_order_usdt, limit_sell_order_usdt_ assert trade.close_date is not None -def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order_usdt_open, +def test_handle_overlapping_signals(default_conf_usdt, ticker_usdt, limit_buy_order_usdt_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(side_effect=[ limit_buy_order_usdt_open, {'id': 1234553382}, @@ -1853,7 +1853,7 @@ def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order_usdt_o get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade, value=(True, True, None)) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) @@ -1894,14 +1894,14 @@ def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order_usdt_o assert freqtrade.handle_trade(trades[0]) is True -def test_handle_trade_roi(default_conf, ticker, limit_buy_order_usdt_open, +def test_handle_trade_roi(default_conf_usdt, ticker_usdt, limit_buy_order_usdt_open, fee, mocker, caplog) -> None: caplog.set_level(logging.DEBUG) patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(side_effect=[ limit_buy_order_usdt_open, {'id': 1234553382}, @@ -1909,7 +1909,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order_usdt_open, get_fee=fee, ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) @@ -1925,18 +1925,18 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order_usdt_open, # if ROI is reached we must sell patch_get_signal(freqtrade, value=(False, True, None)) assert freqtrade.handle_trade(trade) - assert log_has("ETH/BTC - Required profit reached. sell_type=SellType.ROI", + assert log_has("ETH/USDT - Required profit reached. sell_type=SellType.ROI", caplog) -def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_usdt_open, +def test_handle_trade_use_sell_signal(default_conf_usdt, ticker_usdt, limit_buy_order_usdt_open, limit_sell_order_usdt_open, fee, mocker, caplog) -> None: # use_sell_signal is True buy default caplog.set_level(logging.DEBUG) patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(side_effect=[ limit_buy_order_usdt_open, limit_sell_order_usdt_open, @@ -1944,7 +1944,7 @@ def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_usdt get_fee=fee, ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.enter_positions() @@ -1957,21 +1957,21 @@ def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_usdt patch_get_signal(freqtrade, value=(False, True, None)) assert freqtrade.handle_trade(trade) - assert log_has("ETH/BTC - Sell signal received. sell_type=SellType.SELL_SIGNAL", + assert log_has("ETH/USDT - Sell signal received. sell_type=SellType.SELL_SIGNAL", caplog) -def test_close_trade(default_conf, ticker, limit_buy_order_usdt, limit_buy_order_usdt_open, limit_sell_order_usdt, +def test_close_trade(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, limit_buy_order_usdt_open, limit_sell_order_usdt, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) # Create trade and sell it @@ -1988,8 +1988,8 @@ def test_close_trade(default_conf, ticker, limit_buy_order_usdt, limit_buy_order freqtrade.handle_trade(trade) -def test_bot_loop_start_called_once(mocker, default_conf, caplog): - ftbot = get_patched_freqtradebot(mocker, default_conf) +def test_bot_loop_start_called_once(mocker, default_conf_usdt, caplog): + ftbot = get_patched_freqtradebot(mocker, default_conf_usdt) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trade') patch_get_signal(ftbot) ftbot.strategy.bot_loop_start = MagicMock(side_effect=ValueError) @@ -2001,9 +2001,9 @@ def test_bot_loop_start_called_once(mocker, default_conf, caplog): assert ftbot.strategy.analyze.call_count == 1 -def test_check_handle_timedout_buy_usercustom(default_conf, ticker, limit_buy_order_old, open_trade, +def test_check_handle_timedout_buy_usercustom(default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, fee, mocker) -> None: - default_conf["unfilledtimeout"] = {"buy": 1400, "sell": 30} + default_conf_usdt["unfilledtimeout"] = {"buy": 1400, "sell": 30} rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock(return_value=limit_buy_order_old) @@ -2014,13 +2014,13 @@ def test_check_handle_timedout_buy_usercustom(default_conf, ticker, limit_buy_or patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, fetch_order=MagicMock(return_value=limit_buy_order_old), cancel_order_with_result=cancel_order_wr_mock, cancel_order=cancel_order_mock, get_fee=fee ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) Trade.query.session.add(open_trade) @@ -2057,7 +2057,7 @@ def test_check_handle_timedout_buy_usercustom(default_conf, ticker, limit_buy_or assert freqtrade.strategy.check_buy_timeout.call_count == 1 -def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, open_trade, +def test_check_handle_timedout_buy(default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, fee, mocker) -> None: rpc_mock = patch_RPCManager(mocker) limit_buy_cancel = deepcopy(limit_buy_order_old) @@ -2066,12 +2066,12 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, op patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, fetch_order=MagicMock(return_value=limit_buy_order_old), cancel_order_with_result=cancel_order_mock, get_fee=fee ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) Trade.query.session.add(open_trade) @@ -2087,7 +2087,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, op assert freqtrade.strategy.check_buy_timeout.call_count == 0 -def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, open_trade, +def test_check_handle_cancelled_buy(default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, fee, mocker, caplog) -> None: """ Handle Buy order cancelled on exchange""" rpc_mock = patch_RPCManager(mocker) @@ -2096,12 +2096,12 @@ def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, o limit_buy_order_old.update({"status": "canceled", 'filled': 0.0}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, fetch_order=MagicMock(return_value=limit_buy_order_old), cancel_order=cancel_order_mock, get_fee=fee ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) Trade.query.session.add(open_trade) @@ -2115,7 +2115,7 @@ def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, o assert log_has_re("Buy order cancelled on exchange for Trade.*", caplog) -def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_order_old, open_trade, +def test_check_handle_timedout_buy_exception(default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, fee, mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() @@ -2123,12 +2123,12 @@ def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_ord mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, fetch_order=MagicMock(side_effect=ExchangeError), cancel_order=cancel_order_mock, get_fee=fee ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) Trade.query.session.add(open_trade) @@ -2141,19 +2141,19 @@ def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_ord assert nb_trades == 1 -def test_check_handle_timedout_sell_usercustom(default_conf, ticker, limit_sell_order_old, mocker, +def test_check_handle_timedout_sell_usercustom(default_conf_usdt, ticker_usdt, limit_sell_order_old, mocker, open_trade) -> None: - default_conf["unfilledtimeout"] = {"buy": 1440, "sell": 1440} + default_conf_usdt["unfilledtimeout"] = {"buy": 1440, "sell": 1440} rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, fetch_order=MagicMock(return_value=limit_sell_order_old), cancel_order=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) open_trade.open_date = arrow.utcnow().shift(hours=-5).datetime open_trade.close_date = arrow.utcnow().shift(minutes=-601).datetime @@ -2190,18 +2190,18 @@ def test_check_handle_timedout_sell_usercustom(default_conf, ticker, limit_sell_ assert freqtrade.strategy.check_sell_timeout.call_count == 1 -def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker, +def test_check_handle_timedout_sell(default_conf_usdt, ticker_usdt, limit_sell_order_old, mocker, open_trade) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, fetch_order=MagicMock(return_value=limit_sell_order_old), cancel_order=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) open_trade.open_date = arrow.utcnow().shift(hours=-5).datetime open_trade.close_date = arrow.utcnow().shift(minutes=-601).datetime @@ -2220,7 +2220,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, assert freqtrade.strategy.check_sell_timeout.call_count == 0 -def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old, open_trade, +def test_check_handle_cancelled_sell(default_conf_usdt, ticker_usdt, limit_sell_order_old, open_trade, mocker, caplog) -> None: """ Handle sell order cancelled on exchange""" rpc_mock = patch_RPCManager(mocker) @@ -2229,11 +2229,11 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, fetch_order=MagicMock(return_value=limit_sell_order_old), cancel_order_with_result=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) open_trade.open_date = arrow.utcnow().shift(hours=-5).datetime open_trade.close_date = arrow.utcnow().shift(minutes=-601).datetime @@ -2249,7 +2249,7 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old, assert log_has_re("Sell order cancelled on exchange for Trade.*", caplog) -def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, +def test_check_handle_timedout_partial(default_conf_usdt, ticker_usdt, limit_buy_order_old_partial, open_trade, mocker) -> None: rpc_mock = patch_RPCManager(mocker) limit_buy_canceled = deepcopy(limit_buy_order_old_partial) @@ -2259,11 +2259,11 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, fetch_order=MagicMock(return_value=limit_buy_order_old_partial), cancel_order_with_result=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) Trade.query.session.add(open_trade) @@ -2278,7 +2278,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old assert trades[0].stake_amount == open_trade.open_rate * trades[0].amount -def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, caplog, fee, +def test_check_handle_timedout_partial_fee(default_conf_usdt, ticker_usdt, open_trade, caplog, fee, limit_buy_order_old_partial, trades_for_order, limit_buy_order_old_partial_canceled, mocker) -> None: rpc_mock = patch_RPCManager(mocker) @@ -2287,12 +2287,12 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, fetch_order=MagicMock(return_value=limit_buy_order_old_partial), cancel_order_with_result=cancel_order_mock, get_trades_for_order=MagicMock(return_value=trades_for_order), ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) assert open_trade.amount == limit_buy_order_old_partial['amount'] @@ -2317,7 +2317,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap assert pytest.approx(trades[0].fee_open) == 0.001 -def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, caplog, fee, +def test_check_handle_timedout_partial_except(default_conf_usdt, ticker_usdt, open_trade, caplog, fee, limit_buy_order_old_partial, trades_for_order, limit_buy_order_old_partial_canceled, mocker) -> None: rpc_mock = patch_RPCManager(mocker) @@ -2325,14 +2325,14 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, fetch_order=MagicMock(return_value=limit_buy_order_old_partial), cancel_order_with_result=cancel_order_mock, get_trades_for_order=MagicMock(return_value=trades_for_order), ) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', MagicMock(side_effect=DependencyException)) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) assert open_trade.amount == limit_buy_order_old_partial['amount'] @@ -2357,7 +2357,7 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, assert trades[0].fee_open == fee() -def test_check_handle_timedout_exception(default_conf, ticker, open_trade_usdt, mocker, caplog) -> None: +def test_check_handle_timedout_exception(default_conf_usdt, ticker_usdt, open_trade_usdt, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock() @@ -2369,11 +2369,11 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade_usdt, ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, fetch_order=MagicMock(side_effect=ExchangeError('Oh snap')), cancel_order=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) Trade.query.session.add(open_trade_usdt) @@ -2385,7 +2385,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade_usdt, caplog) -def test_handle_cancel_enter(mocker, caplog, default_conf, limit_buy_order_usdt) -> None: +def test_handle_cancel_enter(mocker, caplog, default_conf_usdt, limit_buy_order_usdt) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_buy_order = deepcopy(limit_buy_order_usdt) @@ -2395,7 +2395,7 @@ def test_handle_cancel_enter(mocker, caplog, default_conf, limit_buy_order_usdt) cancel_order_mock = MagicMock(return_value=cancel_buy_order) mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) freqtrade._notify_enter_cancel = MagicMock() trade = MagicMock() @@ -2430,7 +2430,7 @@ def test_handle_cancel_enter(mocker, caplog, default_conf, limit_buy_order_usdt) @pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'ftx', 'kraken', 'bittrex'], indirect=['limit_buy_order_canceled_empty']) -def test_handle_cancel_enter_exchanges(mocker, caplog, default_conf, +def test_handle_cancel_enter_exchanges(mocker, caplog, default_conf_usdt, limit_buy_order_canceled_empty) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -2438,7 +2438,7 @@ def test_handle_cancel_enter_exchanges(mocker, caplog, default_conf, 'freqtrade.exchange.Exchange.cancel_order_with_result', return_value=limit_buy_order_canceled_empty) nofiy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot._notify_enter_cancel') - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) reason = CANCEL_REASON['TIMEOUT'] trade = MagicMock() @@ -2455,7 +2455,7 @@ def test_handle_cancel_enter_exchanges(mocker, caplog, default_conf, 'String Return value', 123 ]) -def test_handle_cancel_enter_corder_empty(mocker, default_conf, limit_buy_order_usdt, +def test_handle_cancel_enter_corder_empty(mocker, default_conf_usdt, limit_buy_order_usdt, cancelorder) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -2465,7 +2465,7 @@ def test_handle_cancel_enter_corder_empty(mocker, default_conf, limit_buy_order_ cancel_order=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) freqtrade._notify_enter_cancel = MagicMock() trade = MagicMock() @@ -2483,7 +2483,7 @@ def test_handle_cancel_enter_corder_empty(mocker, default_conf, limit_buy_order_ assert cancel_order_mock.call_count == 1 -def test_handle_cancel_exit_limit(mocker, default_conf, fee) -> None: +def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: send_msg_mock = patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock() @@ -2493,7 +2493,7 @@ def test_handle_cancel_exit_limit(mocker, default_conf, fee) -> None: ) mocker.patch('freqtrade.exchange.Exchange.get_rate', return_value=0.245441) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) trade = Trade( pair='LTC/ETH', @@ -2528,13 +2528,13 @@ def test_handle_cancel_exit_limit(mocker, default_conf, fee) -> None: assert send_msg_mock.call_count == 1 -def test_handle_cancel_exit_cancel_exception(mocker, default_conf) -> None: +def test_handle_cancel_exit_cancel_exception(mocker, default_conf_usdt) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch( 'freqtrade.exchange.Exchange.cancel_order_with_result', side_effect=InvalidOrderException()) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) trade = MagicMock() reason = CANCEL_REASON['TIMEOUT'] @@ -2544,17 +2544,17 @@ def test_handle_cancel_exit_cancel_exception(mocker, default_conf) -> None: assert freqtrade.handle_cancel_exit(trade, order, reason) == 'error cancelling order' -def test_execute_trade_exit_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> None: +def test_execute_trade_exit_up(default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_up, mocker) -> None: rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, get_fee=fee, _is_dry_limit_order_filled=MagicMock(return_value=False), ) - patch_whitelist(mocker, default_conf) - freqtrade = FreqtradeBot(default_conf) + patch_whitelist(mocker, default_conf_usdt) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) freqtrade.strategy.confirm_trade_exit = MagicMock(return_value=False) @@ -2569,10 +2569,10 @@ def test_execute_trade_exit_up(default_conf, ticker, fee, ticker_sell_up, mocker # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker_sell_up + fetch_ticker=ticker_usdt_sell_up ) # Prevented sell ... - freqtrade.execute_trade_exit(trade=trade, limit=ticker_sell_up()['bid'], + freqtrade.execute_trade_exit(trade=trade, limit=ticker_usdt_sell_up()['bid'], sell_reason=SellCheckTuple(sell_type=SellType.ROI)) assert rpc_mock.call_count == 0 assert freqtrade.strategy.confirm_trade_exit.call_count == 1 @@ -2580,7 +2580,7 @@ def test_execute_trade_exit_up(default_conf, ticker, fee, ticker_sell_up, mocker # Repatch with true freqtrade.strategy.confirm_trade_exit = MagicMock(return_value=True) - freqtrade.execute_trade_exit(trade=trade, limit=ticker_sell_up()['bid'], + freqtrade.execute_trade_exit(trade=trade, limit=ticker_usdt_sell_up()['bid'], sell_reason=SellCheckTuple(sell_type=SellType.ROI)) assert freqtrade.strategy.confirm_trade_exit.call_count == 1 @@ -2590,7 +2590,7 @@ def test_execute_trade_exit_up(default_conf, ticker, fee, ticker_sell_up, mocker 'trade_id': 1, 'type': RPCMessageType.SELL, 'exchange': 'Binance', - 'pair': 'ETH/BTC', + 'pair': 'ETH/USDT', 'gain': 'profit', 'limit': 1.172e-05, 'amount': 91.07468123, @@ -2608,17 +2608,17 @@ def test_execute_trade_exit_up(default_conf, ticker, fee, ticker_sell_up, mocker } == last_msg -def test_execute_trade_exit_down(default_conf, ticker, fee, ticker_sell_down, mocker) -> None: +def test_execute_trade_exit_down(default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_down, mocker) -> None: rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, get_fee=fee, _is_dry_limit_order_filled=MagicMock(return_value=False), ) - patch_whitelist(mocker, default_conf) - freqtrade = FreqtradeBot(default_conf) + patch_whitelist(mocker, default_conf_usdt) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) # Create some test data @@ -2630,10 +2630,10 @@ def test_execute_trade_exit_down(default_conf, ticker, fee, ticker_sell_down, mo # Decrease the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker_sell_down + fetch_ticker=ticker_usdt_sell_down ) - freqtrade.execute_trade_exit(trade=trade, limit=ticker_sell_down()['bid'], + freqtrade.execute_trade_exit(trade=trade, limit=ticker_usdt_sell_down()['bid'], sell_reason=SellCheckTuple(sell_type=SellType.STOP_LOSS)) assert rpc_mock.call_count == 2 @@ -2642,7 +2642,7 @@ def test_execute_trade_exit_down(default_conf, ticker, fee, ticker_sell_down, mo 'type': RPCMessageType.SELL, 'trade_id': 1, 'exchange': 'Binance', - 'pair': 'ETH/BTC', + 'pair': 'ETH/USDT', 'gain': 'loss', 'limit': 1.044e-05, 'amount': 91.07468123, @@ -2660,18 +2660,18 @@ def test_execute_trade_exit_down(default_conf, ticker, fee, ticker_sell_down, mo } == last_msg -def test_execute_trade_exit_custom_exit_price(default_conf, ticker, fee, ticker_sell_up, +def test_execute_trade_exit_custom_exit_price(default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_up, mocker) -> None: rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, get_fee=fee, _is_dry_limit_order_filled=MagicMock(return_value=False), ) - patch_whitelist(mocker, default_conf) - freqtrade = FreqtradeBot(default_conf) + patch_whitelist(mocker, default_conf_usdt) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) freqtrade.strategy.confirm_trade_exit = MagicMock(return_value=False) @@ -2686,7 +2686,7 @@ def test_execute_trade_exit_custom_exit_price(default_conf, ticker, fee, ticker_ # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker_sell_up + fetch_ticker=ticker_usdt_sell_up ) freqtrade.strategy.confirm_trade_exit = MagicMock(return_value=True) @@ -2694,7 +2694,7 @@ def test_execute_trade_exit_custom_exit_price(default_conf, ticker, fee, ticker_ # Set a custom exit price freqtrade.strategy.custom_exit_price = lambda **kwargs: 1.170e-05 - freqtrade.execute_trade_exit(trade=trade, limit=ticker_sell_up()['bid'], + freqtrade.execute_trade_exit(trade=trade, limit=ticker_usdt_sell_up()['bid'], sell_reason=SellCheckTuple(sell_type=SellType.SELL_SIGNAL)) # Sell price must be different to default bid price @@ -2707,7 +2707,7 @@ def test_execute_trade_exit_custom_exit_price(default_conf, ticker, fee, ticker_ 'trade_id': 1, 'type': RPCMessageType.SELL, 'exchange': 'Binance', - 'pair': 'ETH/BTC', + 'pair': 'ETH/USDT', 'gain': 'profit', 'limit': 1.170e-05, 'amount': 91.07468123, @@ -2725,18 +2725,18 @@ def test_execute_trade_exit_custom_exit_price(default_conf, ticker, fee, ticker_ } == last_msg -def test_execute_trade_exit_down_stoploss_on_exchange_dry_run(default_conf, ticker, fee, - ticker_sell_down, mocker) -> None: +def test_execute_trade_exit_down_stoploss_on_exchange_dry_run(default_conf_usdt, ticker_usdt, fee, + ticker_usdt_sell_down, mocker) -> None: rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, get_fee=fee, _is_dry_limit_order_filled=MagicMock(return_value=False), ) - patch_whitelist(mocker, default_conf) - freqtrade = FreqtradeBot(default_conf) + patch_whitelist(mocker, default_conf_usdt) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) # Create some test data @@ -2748,15 +2748,15 @@ def test_execute_trade_exit_down_stoploss_on_exchange_dry_run(default_conf, tick # Decrease the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker_sell_down + fetch_ticker=ticker_usdt_sell_down ) - default_conf['dry_run'] = True + default_conf_usdt['dry_run'] = True freqtrade.strategy.order_types['stoploss_on_exchange'] = True # Setting trade stoploss to 0.01 trade.stop_loss = 0.00001099 * 0.99 - freqtrade.execute_trade_exit(trade=trade, limit=ticker_sell_down()['bid'], + freqtrade.execute_trade_exit(trade=trade, limit=ticker_usdt_sell_down()['bid'], sell_reason=SellCheckTuple(sell_type=SellType.STOP_LOSS)) assert rpc_mock.call_count == 2 @@ -2766,7 +2766,7 @@ def test_execute_trade_exit_down_stoploss_on_exchange_dry_run(default_conf, tick 'type': RPCMessageType.SELL, 'trade_id': 1, 'exchange': 'Binance', - 'pair': 'ETH/BTC', + 'pair': 'ETH/USDT', 'gain': 'loss', 'limit': 1.08801e-05, 'amount': 91.07468123, @@ -2785,8 +2785,8 @@ def test_execute_trade_exit_down_stoploss_on_exchange_dry_run(default_conf, tick def test_execute_trade_exit_sloe_cancel_exception( - mocker, default_conf, ticker, fee, caplog) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker, default_conf_usdt, ticker_usdt, fee, caplog) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order', side_effect=InvalidOrderException()) mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=300)) @@ -2797,7 +2797,7 @@ def test_execute_trade_exit_sloe_cancel_exception( patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, get_fee=fee, create_order=create_order_mock, ) @@ -2818,10 +2818,10 @@ def test_execute_trade_exit_sloe_cancel_exception( assert log_has('Could not cancel stoploss order abcd', caplog) -def test_execute_trade_exit_with_stoploss_on_exchange(default_conf, ticker, fee, ticker_sell_up, +def test_execute_trade_exit_with_stoploss_on_exchange(default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_up, mocker) -> None: - default_conf['exchange']['name'] = 'binance' + default_conf_usdt['exchange']['name'] = 'binance' rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) stoploss = MagicMock(return_value={ @@ -2834,7 +2834,7 @@ def test_execute_trade_exit_with_stoploss_on_exchange(default_conf, ticker, fee, cancel_order = MagicMock(return_value=True) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, get_fee=fee, amount_to_precision=lambda s, x, y: y, price_to_precision=lambda s, x, y: y, @@ -2843,7 +2843,7 @@ def test_execute_trade_exit_with_stoploss_on_exchange(default_conf, ticker, fee, _is_dry_limit_order_filled=MagicMock(side_effect=[True, False]), ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) freqtrade.strategy.order_types['stoploss_on_exchange'] = True patch_get_signal(freqtrade) @@ -2860,10 +2860,10 @@ def test_execute_trade_exit_with_stoploss_on_exchange(default_conf, ticker, fee, # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker_sell_up + fetch_ticker=ticker_usdt_sell_up ) - freqtrade.execute_trade_exit(trade=trade, limit=ticker_sell_up()['bid'], + freqtrade.execute_trade_exit(trade=trade, limit=ticker_usdt_sell_up()['bid'], sell_reason=SellCheckTuple(sell_type=SellType.STOP_LOSS)) trade = Trade.query.first() @@ -2872,14 +2872,14 @@ def test_execute_trade_exit_with_stoploss_on_exchange(default_conf, ticker, fee, assert rpc_mock.call_count == 3 -def test_may_execute_trade_exit_after_stoploss_on_exchange_hit(default_conf, ticker, fee, +def test_may_execute_trade_exit_after_stoploss_on_exchange_hit(default_conf_usdt, ticker_usdt, fee, mocker) -> None: - default_conf['exchange']['name'] = 'binance' + default_conf_usdt['exchange']['name'] = 'binance' rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, get_fee=fee, amount_to_precision=lambda s, x, y: y, price_to_precision=lambda s, x, y: y, @@ -2895,7 +2895,7 @@ def test_may_execute_trade_exit_after_stoploss_on_exchange_hit(default_conf, tic mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) freqtrade.strategy.order_types['stoploss_on_exchange'] = True patch_get_signal(freqtrade) @@ -2944,18 +2944,18 @@ def test_may_execute_trade_exit_after_stoploss_on_exchange_hit(default_conf, tic assert rpc_mock.call_args_list[2][0][0]['type'] == RPCMessageType.SELL -def test_execute_trade_exit_market_order(default_conf, ticker, fee, - ticker_sell_up, mocker) -> None: +def test_execute_trade_exit_market_order(default_conf_usdt, ticker_usdt, fee, + ticker_usdt_sell_up, mocker) -> None: rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, get_fee=fee, _is_dry_limit_order_filled=MagicMock(return_value=False), ) - patch_whitelist(mocker, default_conf) - freqtrade = FreqtradeBot(default_conf) + patch_whitelist(mocker, default_conf_usdt) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) # Create some test data @@ -2967,11 +2967,11 @@ def test_execute_trade_exit_market_order(default_conf, ticker, fee, # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker_sell_up + fetch_ticker=ticker_usdt_sell_up ) freqtrade.config['order_types']['sell'] = 'market' - freqtrade.execute_trade_exit(trade=trade, limit=ticker_sell_up()['bid'], + freqtrade.execute_trade_exit(trade=trade, limit=ticker_usdt_sell_up()['bid'], sell_reason=SellCheckTuple(sell_type=SellType.ROI)) assert not trade.is_open @@ -2983,7 +2983,7 @@ def test_execute_trade_exit_market_order(default_conf, ticker, fee, 'type': RPCMessageType.SELL, 'trade_id': 1, 'exchange': 'Binance', - 'pair': 'ETH/BTC', + 'pair': 'ETH/USDT', 'gain': 'profit', 'limit': 1.172e-05, 'amount': 91.07468123, @@ -3002,13 +3002,13 @@ def test_execute_trade_exit_market_order(default_conf, ticker, fee, } == last_msg -def test_execute_trade_exit_insufficient_funds_error(default_conf, ticker, fee, - ticker_sell_up, mocker) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) +def test_execute_trade_exit_insufficient_funds_error(default_conf_usdt, ticker_usdt, fee, + ticker_usdt_sell_up, mocker) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mock_insuf = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_insufficient_funds') mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, get_fee=fee, create_order=MagicMock(side_effect=[ {'id': 1234553382}, @@ -3026,11 +3026,11 @@ def test_execute_trade_exit_insufficient_funds_error(default_conf, ticker, fee, # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker_sell_up + fetch_ticker=ticker_usdt_sell_up ) sell_reason = SellCheckTuple(sell_type=SellType.ROI) - assert not freqtrade.execute_trade_exit(trade=trade, limit=ticker_sell_up()['bid'], + assert not freqtrade.execute_trade_exit(trade=trade, limit=ticker_usdt_sell_up()['bid'], sell_reason=sell_reason) assert mock_insuf.call_count == 1 @@ -3047,7 +3047,7 @@ def test_execute_trade_exit_insufficient_funds_error(default_conf, ticker, fee, (False, 0.00000172, 0.00000173, True, False, SellType.SELL_SIGNAL.value), ]) def test_sell_profit_only( - default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, + default_conf_usdt, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, mocker, profit_only, bid, ask, handle_first, handle_second, sell_type) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -3064,12 +3064,12 @@ def test_sell_profit_only( ]), get_fee=fee, ) - default_conf.update({ + default_conf_usdt.update({ 'use_sell_signal': True, 'sell_profit_only': profit_only, 'sell_profit_offset': 0.1, }) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) if sell_type == SellType.SELL_SIGNAL.value: freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) @@ -3089,7 +3089,7 @@ def test_sell_profit_only( assert freqtrade.handle_trade(trade) is True -def test_sell_not_enough_balance(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, +def test_sell_not_enough_balance(default_conf_usdt, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -3107,7 +3107,7 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order_usdt, limit_buy_o get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) @@ -3128,7 +3128,7 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order_usdt, limit_buy_o (95.29, False), (91.29, True) ]) -def test__safe_exit_amount(default_conf, fee, caplog, mocker, amount_wallet, has_err): +def test__safe_exit_amount(default_conf_usdt, fee, caplog, mocker, amount_wallet, has_err): patch_RPCManager(mocker) patch_exchange(mocker) amount = 95.33 @@ -3144,7 +3144,7 @@ def test__safe_exit_amount(default_conf, fee, caplog, mocker, amount_wallet, has fee_open=fee.return_value, fee_close=fee.return_value, ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) if has_err: with pytest.raises(DependencyException, match=r"Not enough amount to sell."): @@ -3161,15 +3161,15 @@ def test__safe_exit_amount(default_conf, fee, caplog, mocker, amount_wallet, has assert wallet_update.call_count == 1 -def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplog) -> None: +def test_locked_pairs(default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_down, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) # Create some test data @@ -3181,12 +3181,12 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo # Decrease the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker_sell_down + fetch_ticker=ticker_usdt_sell_down ) - freqtrade.execute_trade_exit(trade=trade, limit=ticker_sell_down()['bid'], + freqtrade.execute_trade_exit(trade=trade, limit=ticker_usdt_sell_down()['bid'], sell_reason=SellCheckTuple(sell_type=SellType.STOP_LOSS)) - trade.close(ticker_sell_down()['bid']) + trade.close(ticker_usdt_sell_down()['bid']) assert freqtrade.strategy.is_pair_locked(trade.pair) # reinit - should buy other pair. @@ -3196,7 +3196,7 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo assert log_has_re(f"Pair {trade.pair} is still locked.*", caplog) -def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, +def test_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -3213,9 +3213,9 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order_usdt, limit_buy_ ]), get_fee=fee, ) - default_conf['ignore_roi_if_buy_signal'] = True + default_conf_usdt['ignore_roi_if_buy_signal'] = True - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) @@ -3233,7 +3233,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order_usdt, limit_buy_ assert trade.sell_reason == SellType.ROI.value -def test_trailing_stop_loss(default_conf, limit_buy_order_usdt_open, limit_buy_order_usdt, +def test_trailing_stop_loss(default_conf_usdt, limit_buy_order_usdt_open, limit_buy_order_usdt, fee, caplog, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -3250,9 +3250,9 @@ def test_trailing_stop_loss(default_conf, limit_buy_order_usdt_open, limit_buy_o ]), get_fee=fee, ) - default_conf['trailing_stop'] = True - patch_whitelist(mocker, default_conf) - freqtrade = FreqtradeBot(default_conf) + default_conf_usdt['trailing_stop'] = True + patch_whitelist(mocker, default_conf_usdt) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) @@ -3260,7 +3260,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order_usdt_open, limit_buy_o trade = Trade.query.first() assert freqtrade.handle_trade(trade) is False - # Raise ticker above buy price + # Raise ticker_usdt above buy price mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': 2.0 * 1.5, @@ -3283,7 +3283,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order_usdt_open, limit_buy_o # Sell as trailing-stop is reached assert freqtrade.handle_trade(trade) is True # TODO: Does this make sense? How is stoploss 2.7? - assert log_has("ETH/BTC - HIT STOP: current price at 2.200000, stoploss is 2.700000, " + assert log_has("ETH/USDT - HIT STOP: current price at 2.200000, stoploss is 2.700000, " "initial stoploss was at 1.800000, trade opened at 2.000000", caplog) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value @@ -3293,7 +3293,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order_usdt_open, limit_buy_o (0.011, False, 2.0394), (0.055, True, 1.8), ]) -def test_trailing_stop_loss_positive(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, +def test_trailing_stop_loss_positive(default_conf_usdt, limit_buy_order_usdt, limit_buy_order_usdt_open, offset, fee, caplog, mocker, trail_if_reached, second_sl) -> None: buy_price = limit_buy_order_usdt['price'] patch_RPCManager(mocker) @@ -3311,14 +3311,14 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order_usdt, limit_b ]), get_fee=fee, ) - default_conf['trailing_stop'] = True - default_conf['trailing_stop_positive'] = 0.01 + default_conf_usdt['trailing_stop'] = True + default_conf_usdt['trailing_stop_positive'] = 0.01 if offset: - default_conf['trailing_stop_positive_offset'] = offset - default_conf['trailing_only_offset_is_reached'] = trail_if_reached - patch_whitelist(mocker, default_conf) + default_conf_usdt['trailing_stop_positive_offset'] = offset + default_conf_usdt['trailing_only_offset_is_reached'] = trail_if_reached + patch_whitelist(mocker, default_conf_usdt) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.enter_positions() @@ -3329,7 +3329,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order_usdt, limit_b # stop-loss not reached assert freqtrade.handle_trade(trade) is False - # Raise ticker above buy price + # Raise ticker_usdt above buy price mocker.patch( 'freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ @@ -3341,13 +3341,13 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order_usdt, limit_b # stop-loss not reached, adjusted stoploss assert freqtrade.handle_trade(trade) is False # TODO: is 0.0249% correct? Shouldn't it be higher? - caplog_text = f"ETH/BTC - Using positive stoploss: 0.01 offset: {offset} profit: 0.0249%" + caplog_text = f"ETH/USDT - Using positive stoploss: 0.01 offset: {offset} profit: 0.0249%" if trail_if_reached: assert not log_has(caplog_text, caplog) - assert not log_has("ETH/BTC - Adjusting stoploss...", caplog) + assert not log_has("ETH/USDT - Adjusting stoploss...", caplog) else: assert log_has(caplog_text, caplog) - assert log_has("ETH/BTC - Adjusting stoploss...", caplog) + assert log_has("ETH/USDT - Adjusting stoploss...", caplog) assert trade.stop_loss == second_sl caplog.clear() @@ -3361,10 +3361,10 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order_usdt, limit_b ) assert freqtrade.handle_trade(trade) is False assert log_has( - f"ETH/BTC - Using positive stoploss: 0.01 offset: {offset} profit: 0.0572%", + f"ETH/USDT - Using positive stoploss: 0.01 offset: {offset} profit: 0.0572%", caplog ) - assert log_has("ETH/BTC - Adjusting stoploss...", caplog) + assert log_has("ETH/USDT - Adjusting stoploss...", caplog) mocker.patch( 'freqtrade.exchange.Exchange.fetch_ticker', @@ -3377,13 +3377,13 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order_usdt, limit_b # Lower price again (but still positive) assert freqtrade.handle_trade(trade) is True assert log_has( - f"ETH/BTC - HIT STOP: current price at {buy_price + 0.02:.6f}, " + f"ETH/USDT - HIT STOP: current price at {buy_price + 0.02:.6f}, " f"stoploss is {trade.stop_loss:.6f}, " f"initial stoploss was at 1.800000, trade opened at 2.000000", caplog) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value -def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order_usdt, limit_buy_order_usdt_open, +def test_disable_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usdt, limit_buy_order_usdt_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -3402,10 +3402,10 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order_usdt, li get_fee=fee, _is_dry_limit_order_filled=MagicMock(return_value=False), ) - default_conf['ask_strategy'] = { + default_conf_usdt['ask_strategy'] = { 'ignore_roi_if_buy_signal': False } - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) @@ -3423,7 +3423,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order_usdt, li assert trade.sell_reason == SellType.SELL_SIGNAL.value -def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fee, caplog, mocker): +def test_get_real_amount_quote(default_conf_usdt, trades_for_order, buy_order_fee, fee, caplog, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -3435,7 +3435,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe fee_close=fee.return_value, open_order_id="123456" ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) @@ -3444,7 +3444,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe caplog) -def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fee, fee, +def test_get_real_amount_quote_dust(default_conf_usdt, trades_for_order, buy_order_fee, fee, caplog, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) walletmock = mocker.patch('freqtrade.wallets.Wallets.update') @@ -3459,7 +3459,7 @@ def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fe fee_close=fee.return_value, open_order_id="123456" ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) walletmock.reset_mock() # Amount is kept as is @@ -3469,7 +3469,7 @@ def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fe '- Eating Fee 0.008 into dust', caplog) -def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, fee): +def test_get_real_amount_no_trade(default_conf_usdt, buy_order_fee, caplog, mocker, fee): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) amount = buy_order_fee['amount'] @@ -3482,7 +3482,7 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f fee_close=fee.return_value, open_order_id="123456" ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -3492,7 +3492,7 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f @pytest.mark.parametrize( - 'fee_par,fee_reduction_amount,use_ticker_rate,expected_log', [ + 'fee_par,fee_reduction_amount,use_ticker_usdt_rate,expected_log', [ # basic, amount does not change ({'cost': 0.008, 'currency': 'ETH'}, 0, False, None), # no currency in fee @@ -3511,8 +3511,8 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f ({'cost': 0.008, 'currency': None}, 0, True, None), ]) def test_get_real_amount( - default_conf, trades_for_order, buy_order_fee, fee, mocker, caplog, - fee_par, fee_reduction_amount, use_ticker_rate, expected_log + default_conf_usdt, trades_for_order, buy_order_fee, fee, mocker, caplog, + fee_par, fee_reduction_amount, use_ticker_usdt_rate, expected_log ): buy_order = deepcopy(buy_order_fee) @@ -3530,9 +3530,9 @@ def test_get_real_amount( open_rate=0.245441, open_order_id="123456" ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) - if not use_ticker_rate: + if not use_ticker_usdt_rate: mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', side_effect=ExchangeError) caplog.clear() @@ -3550,7 +3550,7 @@ def test_get_real_amount( (0.02, 'BNB', 0.0005, 0.001518575, 7.996), ]) def test_get_real_amount_multi( - default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker, markets, + default_conf_usdt, trades_for_order2, buy_order_fee, caplog, fee, mocker, markets, fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount, ): @@ -3562,7 +3562,7 @@ def test_get_real_amount_multi( mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = float(sum(x['amount'] for x in trades_for_order)) - default_conf['stake_currency'] = "ETH" + default_conf_usdt['stake_currency'] = "ETH" trade = Trade( pair='LTC/ETH', @@ -3575,8 +3575,8 @@ def test_get_real_amount_multi( ) # Fake markets entry to enable fee parsing - markets['BNB/ETH'] = markets['ETH/BTC'] - freqtrade = get_patched_freqtradebot(mocker, default_conf) + markets['BNB/ETH'] = markets['ETH/USDT'] + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value={'ask': 0.19, 'last': 0.2}) @@ -3600,7 +3600,7 @@ def test_get_real_amount_multi( assert trade.fee_close_currency is None -def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, fee, mocker): +def test_get_real_amount_invalid_order(default_conf_usdt, trades_for_order, buy_order_fee, fee, mocker): limit_buy_order_usdt = deepcopy(buy_order_fee) limit_buy_order_usdt['fee'] = {'cost': 0.004} @@ -3615,13 +3615,13 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order open_rate=0.245441, open_order_id="123456" ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) # Amount does not change assert freqtrade.get_real_amount(trade, limit_buy_order_usdt) == amount -def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_fee, fee, mocker): +def test_get_real_amount_wrong_amount(default_conf_usdt, trades_for_order, buy_order_fee, fee, mocker): limit_buy_order_usdt = deepcopy(buy_order_fee) limit_buy_order_usdt['amount'] = limit_buy_order_usdt['amount'] - 0.001 @@ -3636,14 +3636,14 @@ def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_ fee_close=fee.return_value, open_order_id="123456" ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) # Amount does not change with pytest.raises(DependencyException, match=r"Half bought\? Amounts don't match"): freqtrade.get_real_amount(trade, limit_buy_order_usdt) -def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, buy_order_fee, fee, +def test_get_real_amount_wrong_amount_rounding(default_conf_usdt, trades_for_order, buy_order_fee, fee, mocker): # Floats should not be compared directly. limit_buy_order_usdt = deepcopy(buy_order_fee) @@ -3660,14 +3660,14 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b open_rate=0.245441, open_order_id="123456" ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) # Amount changes by fee amount. assert isclose(freqtrade.get_real_amount(trade, limit_buy_order_usdt), amount - (amount * 0.001), abs_tol=MATH_CLOSE_PREC,) -def test_get_real_amount_open_trade(default_conf, fee, mocker): +def test_get_real_amount_open_trade(default_conf_usdt, fee, mocker): amount = 12345 trade = Trade( pair='LTC/ETH', @@ -3684,7 +3684,7 @@ def test_get_real_amount_open_trade(default_conf, fee, mocker): 'status': 'open', 'side': 'buy', } - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) assert freqtrade.get_real_amount(trade, order) == amount @@ -3696,7 +3696,7 @@ def test_get_real_amount_open_trade(default_conf, fee, mocker): (8.0, 0.1, 8.0, 8.0), (8.0, 0.1, 7.9, 7.9), ]) -def test_apply_fee_conditional(default_conf, fee, caplog, mocker, +def test_apply_fee_conditional(default_conf_usdt, fee, caplog, mocker, amount, fee_abs, wallet, amount_exp): walletmock = mocker.patch('freqtrade.wallets.Wallets.update') mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=wallet) @@ -3709,7 +3709,7 @@ def test_apply_fee_conditional(default_conf, fee, caplog, mocker, fee_close=fee.return_value, open_order_id="123456" ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) walletmock.reset_mock() # Amount is kept as is @@ -3721,23 +3721,23 @@ def test_apply_fee_conditional(default_conf, fee, caplog, mocker, (0.1, False), (100, True), ]) -def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_usdt_open, limit_buy_order_usdt, +def test_order_book_depth_of_market(default_conf_usdt, ticker_usdt, limit_buy_order_usdt_open, limit_buy_order_usdt, fee, mocker, order_book_l2, delta, is_high_delta): - default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True - default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = delta + default_conf_usdt['bid_strategy']['check_depth_of_market']['enabled'] = True + default_conf_usdt['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = delta patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) # Save state of current whitelist - whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) - freqtrade = FreqtradeBot(default_conf) + whitelist = deepcopy(default_conf_usdt['exchange']['pair_whitelist']) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) freqtrade.enter_positions() @@ -3746,7 +3746,7 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_usdt_o assert trade is None else: assert trade is not None - assert trade.stake_amount == 0.001 + assert trade.stake_amount == 10.0 assert trade.is_open assert trade.open_date is not None assert trade.exchange == 'binance' @@ -3757,43 +3757,43 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_usdt_o trade.update(limit_buy_order_usdt) assert trade.open_rate == 2.0 - assert whitelist == default_conf['exchange']['pair_whitelist'] + assert whitelist == default_conf_usdt['exchange']['pair_whitelist'] @pytest.mark.parametrize('exception_thrown,ask,last,order_book_top,order_book', [ (False, 0.045, 0.046, 2, None), (True, 0.042, 0.046, 1, {'bids': [[]], 'asks': [[]]}) ]) -def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, exception_thrown, +def test_order_book_bid_strategy1(mocker, default_conf_usdt, order_book_l2, exception_thrown, ask, last, order_book_top, order_book, caplog) -> None: """ test if function get_rate will return the order book price instead of the ask rate """ patch_exchange(mocker) - ticker_mock = MagicMock(return_value={'ask': ask, 'last': last}) + ticker_usdt_mock = MagicMock(return_value={'ask': ask, 'last': last}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_l2_order_book=MagicMock(return_value=order_book) if order_book else order_book_l2, - fetch_ticker=ticker_mock, + fetch_ticker=ticker_usdt_mock, ) - default_conf['exchange']['name'] = 'binance' - default_conf['bid_strategy']['use_order_book'] = True - default_conf['bid_strategy']['order_book_top'] = order_book_top - default_conf['bid_strategy']['ask_last_balance'] = 0 - default_conf['telegram']['enabled'] = False + default_conf_usdt['exchange']['name'] = 'binance' + default_conf_usdt['bid_strategy']['use_order_book'] = True + default_conf_usdt['bid_strategy']['order_book_top'] = order_book_top + default_conf_usdt['bid_strategy']['ask_last_balance'] = 0 + default_conf_usdt['telegram']['enabled'] = False - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) if exception_thrown: with pytest.raises(PricingError): - freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") + freqtrade.exchange.get_rate('ETH/USDT', refresh=True, side="buy") assert log_has_re( r'Buy Price at location 1 from orderbook could not be determined.', caplog) else: - assert freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") == 0.043935 - assert ticker_mock.call_count == 0 + assert freqtrade.exchange.get_rate('ETH/USDT', refresh=True, side="buy") == 0.043935 + assert ticker_usdt_mock.call_count == 0 -def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: +def test_check_depth_of_market_buy(default_conf_usdt, mocker, order_book_l2) -> None: """ test check depth of market """ @@ -3802,27 +3802,27 @@ def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: 'freqtrade.exchange.Exchange', fetch_l2_order_book=order_book_l2 ) - default_conf['telegram']['enabled'] = False - default_conf['exchange']['name'] = 'binance' - default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True + default_conf_usdt['telegram']['enabled'] = False + default_conf_usdt['exchange']['name'] = 'binance' + default_conf_usdt['bid_strategy']['check_depth_of_market']['enabled'] = True # delta is 100 which is impossible to reach. hence function will return false - default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100 - freqtrade = FreqtradeBot(default_conf) + default_conf_usdt['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100 + freqtrade = FreqtradeBot(default_conf_usdt) - conf = default_conf['bid_strategy']['check_depth_of_market'] - assert freqtrade._check_depth_of_market_buy('ETH/BTC', conf) is False + conf = default_conf_usdt['bid_strategy']['check_depth_of_market'] + assert freqtrade._check_depth_of_market_buy('ETH/USDT', conf) is False -def test_order_book_ask_strategy(default_conf, limit_buy_order_usdt_open, limit_buy_order_usdt, fee, +def test_order_book_ask_strategy(default_conf_usdt, limit_buy_order_usdt_open, limit_buy_order_usdt, fee, limit_sell_order_usdt_open, mocker, order_book_l2, caplog) -> None: """ test order book ask strategy """ mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) - default_conf['exchange']['name'] = 'binance' - default_conf['ask_strategy']['use_order_book'] = True - default_conf['ask_strategy']['order_book_top'] = 1 - default_conf['telegram']['enabled'] = False + default_conf_usdt['exchange']['name'] = 'binance' + default_conf_usdt['ask_strategy']['use_order_book'] = True + default_conf_usdt['ask_strategy']['order_book_top'] = 1 + default_conf_usdt['telegram']['enabled'] = False patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -3838,7 +3838,7 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order_usdt_open, limit_ ]), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) freqtrade.enter_positions() @@ -3863,22 +3863,22 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order_usdt_open, limit_ caplog) -def test_startup_state(default_conf, mocker): - default_conf['pairlist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 20} - } +def test_startup_state(default_conf_usdt, mocker): + default_conf_usdt['pairlist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 20} + } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - worker = get_patched_worker(mocker, default_conf) + worker = get_patched_worker(mocker, default_conf_usdt) assert worker.freqtrade.state is State.RUNNING -def test_startup_trade_reinit(default_conf, edge_conf, mocker): +def test_startup_trade_reinit(default_conf_usdt, edge_conf, mocker): mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) reinit_mock = MagicMock() mocker.patch('freqtrade.persistence.Trade.stoploss_reinitialization', reinit_mock) - ftbot = get_patched_freqtradebot(mocker, default_conf) + ftbot = get_patched_freqtradebot(mocker, default_conf_usdt) ftbot.startup() assert reinit_mock.call_count == 1 @@ -3890,21 +3890,21 @@ def test_startup_trade_reinit(default_conf, edge_conf, mocker): @pytest.mark.usefixtures("init_persistence") -def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order_usdt_open, caplog): - default_conf['dry_run'] = True +def test_sync_wallet_dry_run(mocker, default_conf_usdt, ticker_usdt, fee, limit_buy_order_usdt_open, caplog): + default_conf_usdt['dry_run'] = True # Initialize to 2 times stake amount - default_conf['dry_run_wallet'] = 0.002 - default_conf['max_open_trades'] = 2 - default_conf['tradable_balance_ratio'] = 1.0 + default_conf_usdt['dry_run_wallet'] = 0.002 + default_conf_usdt['max_open_trades'] = 2 + default_conf_usdt['tradable_balance_ratio'] = 1.0 patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, + fetch_ticker=ticker_usdt, create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) - bot = get_patched_freqtradebot(mocker, default_conf) + bot = get_patched_freqtradebot(mocker, default_conf_usdt) patch_get_signal(bot) assert bot.wallets.get_free('BTC') == 0.002 @@ -3922,15 +3922,15 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order_ @pytest.mark.usefixtures("init_persistence") -def test_cancel_all_open_orders(mocker, default_conf, fee, limit_buy_order_usdt, limit_sell_order_usdt): - default_conf['cancel_open_orders_on_exit'] = True +def test_cancel_all_open_orders(mocker, default_conf_usdt, fee, limit_buy_order_usdt, limit_sell_order_usdt): + default_conf_usdt['cancel_open_orders_on_exit'] = True mocker.patch('freqtrade.exchange.Exchange.fetch_order', side_effect=[ ExchangeError(), limit_sell_order_usdt, limit_buy_order_usdt, limit_sell_order_usdt]) buy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_enter') sell_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_exit') - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) create_mock_trades(fee) trades = Trade.query.all() assert len(trades) == MOCK_TRADE_COUNT @@ -3940,8 +3940,8 @@ def test_cancel_all_open_orders(mocker, default_conf, fee, limit_buy_order_usdt, @pytest.mark.usefixtures("init_persistence") -def test_check_for_open_trades(mocker, default_conf, fee): - freqtrade = get_patched_freqtradebot(mocker, default_conf) +def test_check_for_open_trades(mocker, default_conf_usdt, fee): + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) freqtrade.check_for_open_trades() assert freqtrade.rpc.send_msg.call_count == 0 @@ -3956,8 +3956,8 @@ def test_check_for_open_trades(mocker, default_conf, fee): @pytest.mark.usefixtures("init_persistence") -def test_startup_update_open_orders(mocker, default_conf, fee, caplog): - freqtrade = get_patched_freqtradebot(mocker, default_conf) +def test_startup_update_open_orders(mocker, default_conf_usdt, fee, caplog): + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) create_mock_trades(fee) freqtrade.startup_update_open_orders() @@ -3982,8 +3982,8 @@ def test_startup_update_open_orders(mocker, default_conf, fee, caplog): @pytest.mark.usefixtures("init_persistence") -def test_update_closed_trades_without_assigned_fees(mocker, default_conf, fee): - freqtrade = get_patched_freqtradebot(mocker, default_conf) +def test_update_closed_trades_without_assigned_fees(mocker, default_conf_usdt, fee): + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) def patch_with_fee(order): order.update({'fee': {'cost': 0.1, 'rate': 0.01, @@ -4081,8 +4081,8 @@ def test_reupdate_enter_order_fees(mocker, default_conf_usdt, fee, caplog): @pytest.mark.usefixtures("init_persistence") -def test_handle_insufficient_funds(mocker, default_conf, fee): - freqtrade = get_patched_freqtradebot(mocker, default_conf) +def test_handle_insufficient_funds(mocker, default_conf_usdt, fee): + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mock_rlo = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.refind_lost_order') mock_bof = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.reupdate_enter_order_fees') create_mock_trades(fee) @@ -4119,9 +4119,9 @@ def test_handle_insufficient_funds(mocker, default_conf, fee): @pytest.mark.usefixtures("init_persistence") -def test_refind_lost_order(mocker, default_conf, fee, caplog): +def test_refind_lost_order(mocker, default_conf_usdt, fee, caplog): caplog.set_level(logging.DEBUG) - freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mock_uts = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.update_trade_state') mock_fo = mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order', @@ -4217,10 +4217,10 @@ def test_refind_lost_order(mocker, default_conf, fee, caplog): assert log_has(f"Error updating {order['id']}.", caplog) -def test_get_valid_price(mocker, default_conf) -> None: +def test_get_valid_price(mocker, default_conf_usdt) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(default_conf_usdt) freqtrade.config['custom_price_max_distance_ratio'] = 0.02 custom_price_string = "10" From 43339f1660557f6538f05addb464a502ac074b5f Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 26 Sep 2021 13:43:01 -0600 Subject: [PATCH 173/239] A lot of the usdt freqtradebot tests pass now --- tests/conftest.py | 8 ++-- tests/test_freqtradebot.py | 90 ++++++++++++++++++++------------------ 2 files changed, 51 insertions(+), 47 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f2da68dd2..4ef0dfcb4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -441,7 +441,7 @@ def ticker_sell_down(): def ticker_usdt(): return MagicMock(return_value={ 'bid': 2.0, - 'ask': 2.01, + 'ask': 2.1, 'last': 2.0, }) @@ -449,9 +449,9 @@ def ticker_usdt(): @pytest.fixture def ticker_usdt_sell_up(): return MagicMock(return_value={ - 'bid': 2.19, - 'ask': 2.2, - 'last': 2.19, + 'bid': 2.2, + 'ask': 2.3, + 'last': 2.2, }) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 2a96783bc..07f1288ed 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -320,8 +320,8 @@ def test_create_trade_no_stake_amount(default_conf_usdt, ticker_usdt, limit_buy_ @pytest.mark.parametrize('stake_amount,create,amount_enough,max_open_trades', [ - (0.0005, True, True, 99), - (0.000000005, True, False, 99), + (5.0, True, True, 99), + (0.00005, True, False, 99), (0, False, True, 99), (UNLIMITED_STAKE_AMOUNT, False, True, 0), ]) @@ -2592,14 +2592,15 @@ def test_execute_trade_exit_up(default_conf_usdt, ticker_usdt, fee, ticker_usdt_ 'exchange': 'Binance', 'pair': 'ETH/USDT', 'gain': 'profit', - 'limit': 1.172e-05, - 'amount': 91.07468123, + 'limit': 2.2, + 'amount': 5.0, 'order_type': 'limit', - 'open_rate': 1.098e-05, - 'current_rate': 1.173e-05, - 'profit_amount': 6.223e-05, - 'profit_ratio': 0.0620716, - 'stake_currency': 'BTC', + 'open_rate': 2.0, + 'current_rate': 2.3, + # TODO: Double check that profit_amount and profit_ratio are correct + 'profit_amount': 0.9475, + 'profit_ratio': 0.09451372, + 'stake_currency': 'USDT', 'fiat_currency': 'USD', 'sell_reason': SellType.ROI.value, 'open_date': ANY, @@ -2638,20 +2639,21 @@ def test_execute_trade_exit_down(default_conf_usdt, ticker_usdt, fee, ticker_usd assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] + # TODO: Should be a loss, but comes out as a gain assert { 'type': RPCMessageType.SELL, 'trade_id': 1, 'exchange': 'Binance', 'pair': 'ETH/USDT', 'gain': 'loss', - 'limit': 1.044e-05, - 'amount': 91.07468123, + 'limit': 2.01, + 'amount': 5.0, 'order_type': 'limit', - 'open_rate': 1.098e-05, - 'current_rate': 1.043e-05, - 'profit_amount': -5.406e-05, - 'profit_ratio': -0.05392257, - 'stake_currency': 'BTC', + 'open_rate': 2.0, + 'current_rate': 2.0, + 'profit_amount': -0.000125, + 'profit_ratio': -1.247e-05, + 'stake_currency': 'USDT', 'fiat_currency': 'USD', 'sell_reason': SellType.STOP_LOSS.value, 'open_date': ANY, @@ -2692,7 +2694,7 @@ def test_execute_trade_exit_custom_exit_price(default_conf_usdt, ticker_usdt, fe freqtrade.strategy.confirm_trade_exit = MagicMock(return_value=True) # Set a custom exit price - freqtrade.strategy.custom_exit_price = lambda **kwargs: 1.170e-05 + freqtrade.strategy.custom_exit_price = lambda **kwargs: 2.25 freqtrade.execute_trade_exit(trade=trade, limit=ticker_usdt_sell_up()['bid'], sell_reason=SellCheckTuple(sell_type=SellType.SELL_SIGNAL)) @@ -2709,14 +2711,14 @@ def test_execute_trade_exit_custom_exit_price(default_conf_usdt, ticker_usdt, fe 'exchange': 'Binance', 'pair': 'ETH/USDT', 'gain': 'profit', - 'limit': 1.170e-05, - 'amount': 91.07468123, + 'limit': 2.25, + 'amount': 5.0, 'order_type': 'limit', - 'open_rate': 1.098e-05, - 'current_rate': 1.173e-05, + 'open_rate': 2.0, + 'current_rate': 2.3, 'profit_amount': 6.041e-05, - 'profit_ratio': 0.06025919, - 'stake_currency': 'BTC', + 'profit_ratio': 0.07262344, + 'stake_currency': 'USDT', 'fiat_currency': 'USD', 'sell_reason': SellType.SELL_SIGNAL.value, 'open_date': ANY, @@ -2755,27 +2757,28 @@ def test_execute_trade_exit_down_stoploss_on_exchange_dry_run(default_conf_usdt, freqtrade.strategy.order_types['stoploss_on_exchange'] = True # Setting trade stoploss to 0.01 - trade.stop_loss = 0.00001099 * 0.99 + trade.stop_loss = 2.0 * 0.99 freqtrade.execute_trade_exit(trade=trade, limit=ticker_usdt_sell_down()['bid'], sell_reason=SellCheckTuple(sell_type=SellType.STOP_LOSS)) assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] + # TODO: Are these values correct? assert { 'type': RPCMessageType.SELL, 'trade_id': 1, 'exchange': 'Binance', 'pair': 'ETH/USDT', 'gain': 'loss', - 'limit': 1.08801e-05, - 'amount': 91.07468123, + 'limit': 1.98, + 'amount': 5.0, 'order_type': 'limit', - 'open_rate': 1.098e-05, - 'current_rate': 1.043e-05, - 'profit_amount': -1.408e-05, - 'profit_ratio': -0.01404051, - 'stake_currency': 'BTC', + 'open_rate': 2.0, + 'current_rate': 2.0, + 'profit_amount': -0.14975, + 'profit_ratio': -0.01493766, + 'stake_currency': 'USDT', 'fiat_currency': 'USD', 'sell_reason': SellType.STOP_LOSS.value, 'open_date': ANY, @@ -2975,24 +2978,25 @@ def test_execute_trade_exit_market_order(default_conf_usdt, ticker_usdt, fee, sell_reason=SellCheckTuple(sell_type=SellType.ROI)) assert not trade.is_open - assert trade.close_profit == 0.0620716 + assert trade.close_profit == 0.09451372 # TODO: Check this is correct assert rpc_mock.call_count == 3 last_msg = rpc_mock.call_args_list[-1][0][0] + # TODO: Is this correct? assert { 'type': RPCMessageType.SELL, 'trade_id': 1, 'exchange': 'Binance', 'pair': 'ETH/USDT', 'gain': 'profit', - 'limit': 1.172e-05, - 'amount': 91.07468123, + 'limit': 2.2, + 'amount': 5.0, 'order_type': 'market', - 'open_rate': 1.098e-05, - 'current_rate': 1.173e-05, - 'profit_amount': 6.223e-05, - 'profit_ratio': 0.0620716, - 'stake_currency': 'BTC', + 'open_rate': 2.0, + 'current_rate': 2.3, + 'profit_amount': 0.9475, + 'profit_ratio': 0.09451372, + 'stake_currency': 'USDT', 'fiat_currency': 'USD', 'sell_reason': SellType.ROI.value, 'open_date': ANY, @@ -3893,7 +3897,7 @@ def test_startup_trade_reinit(default_conf_usdt, edge_conf, mocker): def test_sync_wallet_dry_run(mocker, default_conf_usdt, ticker_usdt, fee, limit_buy_order_usdt_open, caplog): default_conf_usdt['dry_run'] = True # Initialize to 2 times stake amount - default_conf_usdt['dry_run_wallet'] = 0.002 + default_conf_usdt['dry_run_wallet'] = 20.0 default_conf_usdt['max_open_trades'] = 2 default_conf_usdt['tradable_balance_ratio'] = 1.0 patch_exchange(mocker) @@ -3906,7 +3910,7 @@ def test_sync_wallet_dry_run(mocker, default_conf_usdt, ticker_usdt, fee, limit_ bot = get_patched_freqtradebot(mocker, default_conf_usdt) patch_get_signal(bot) - assert bot.wallets.get_free('BTC') == 0.002 + assert bot.wallets.get_free('USDT') == 20.0 n = bot.enter_positions() assert n == 2 @@ -3916,8 +3920,8 @@ def test_sync_wallet_dry_run(mocker, default_conf_usdt, ticker_usdt, fee, limit_ bot.config['max_open_trades'] = 3 n = bot.enter_positions() assert n == 0 - assert log_has_re(r"Unable to create trade for XRP/BTC: " - r"Available balance \(0.0 BTC\) is lower than stake amount \(0.001 BTC\)", + assert log_has_re(r"Unable to create trade for XRP/USDT: " + r"Available balance \(0.0 USDT\) is lower than stake amount \(10.0 USDT\)", caplog) From 2ee87f8c66cb4b998ba986acd2dfc8729ece5e8f Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Thu, 30 Sep 2021 10:00:56 +0300 Subject: [PATCH 174/239] Fix failing USDT tests due to not enough open markets. --- tests/conftest.py | 78 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4ef0dfcb4..9c500e05c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -387,7 +387,8 @@ def get_default_conf_usdt(testdatadir): "ETH/USDT", "LTC/USDT", "XRP/USDT", - "NEO/USDT" + "NEO/USDT", + "TKN/USDT", ], "pair_blacklist": [ "DOGE/USDT", @@ -696,6 +697,81 @@ def get_markets(): }, 'info': {}, }, + 'XRP/USDT': { + 'id': 'xrpusdt', + 'symbol': 'XRP/USDT', + 'base': 'XRP', + 'quote': 'USDT', + 'active': True, + 'precision': { + 'price': 8, + 'amount': 8, + 'cost': 8, + }, + 'lot': 0.00000001, + 'limits': { + 'amount': { + 'min': 0.01, + 'max': 1000, + }, + 'price': 500000, + 'cost': { + 'min': 0.0001, + 'max': 500000, + }, + }, + 'info': {}, + }, + 'NEO/USDT': { + 'id': 'neousdt', + 'symbol': 'NEO/USDT', + 'base': 'NEO', + 'quote': 'USDT', + 'active': True, + 'precision': { + 'price': 8, + 'amount': 8, + 'cost': 8, + }, + 'lot': 0.00000001, + 'limits': { + 'amount': { + 'min': 0.01, + 'max': 1000, + }, + 'price': 500000, + 'cost': { + 'min': 0.0001, + 'max': 500000, + }, + }, + 'info': {}, + }, + 'TKN/USDT': { + 'id': 'tknusdt', + 'symbol': 'TKN/USDT', + 'base': 'TKN', + 'quote': 'USDT', + 'active': True, + 'precision': { + 'price': 8, + 'amount': 8, + 'cost': 8, + }, + 'lot': 0.00000001, + 'limits': { + 'amount': { + 'min': 0.01, + 'max': 1000, + }, + 'price': 500000, + 'cost': { + 'min': 0.0001, + 'max': 500000, + }, + }, + 'info': {}, + }, 'LTC/USD': { 'id': 'USD-LTC', 'symbol': 'LTC/USD', From 89613702697de80c1ecbd3f7cd75271b4116dea8 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Thu, 30 Sep 2021 10:01:25 +0300 Subject: [PATCH 175/239] Fix failing test due to not updated expected values. --- tests/test_freqtradebot.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 07f1288ed..579dc1b43 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2672,8 +2672,10 @@ def test_execute_trade_exit_custom_exit_price(default_conf_usdt, ticker_usdt, fe get_fee=fee, _is_dry_limit_order_filled=MagicMock(return_value=False), ) - patch_whitelist(mocker, default_conf_usdt) - freqtrade = FreqtradeBot(default_conf_usdt) + config = deepcopy(default_conf_usdt) + config['custom_price_max_distance_ratio'] = 0.1 + patch_whitelist(mocker, config) + freqtrade = FreqtradeBot(config) patch_get_signal(freqtrade) freqtrade.strategy.confirm_trade_exit = MagicMock(return_value=False) @@ -2716,8 +2718,8 @@ def test_execute_trade_exit_custom_exit_price(default_conf_usdt, ticker_usdt, fe 'order_type': 'limit', 'open_rate': 2.0, 'current_rate': 2.3, - 'profit_amount': 6.041e-05, - 'profit_ratio': 0.07262344, + 'profit_amount': 1.196875, + 'profit_ratio': 0.11938903, 'stake_currency': 'USDT', 'fiat_currency': 'USD', 'sell_reason': SellType.SELL_SIGNAL.value, From c820db4c60f6973554e232b52c06ed6e726dd8d0 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Thu, 30 Sep 2021 11:12:15 +0300 Subject: [PATCH 176/239] Fix couple more usdt tests which failed due to ticker prices causing roi being hit, but tests did not expect that to happen. --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9c500e05c..0e5e7c933 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -442,7 +442,7 @@ def ticker_sell_down(): def ticker_usdt(): return MagicMock(return_value={ 'bid': 2.0, - 'ask': 2.1, + 'ask': 2.02, 'last': 2.0, }) From 107fa911a5ef371e675a5d679ab511f1a93579b5 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Thu, 30 Sep 2021 02:34:00 -0600 Subject: [PATCH 177/239] Fixed test_tsl_on_exchange_compatible_with_edge --- tests/test_freqtradebot.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 579dc1b43..bfa3ac353 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1483,9 +1483,9 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 1.9, + 'bid': 2.19, 'ask': 2.2, - 'last': 1.9 + 'last': 2.19 }), create_order=MagicMock(side_effect=[ {'id': limit_buy_order_usdt['id']}, @@ -1540,7 +1540,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, # stoploss initially at 20% as edge dictated it. assert freqtrade.handle_trade(trade) is False assert freqtrade.handle_stoploss_on_exchange(trade) is False - assert trade.stop_loss == 2.178 + assert isclose(trade.stop_loss, 1.76) cancel_order_mock = MagicMock() stoploss_order_mock = MagicMock() @@ -1549,15 +1549,15 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, # price goes down 5% mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ - 'bid': 1.9 * 0.95, + 'bid': 2.19 * 0.95, 'ask': 2.2 * 0.95, - 'last': 1.9 * 0.95 + 'last': 2.19 * 0.95 })) assert freqtrade.handle_trade(trade) is False assert freqtrade.handle_stoploss_on_exchange(trade) is False # stoploss should remain the same - assert trade.stop_loss == 2.178 + assert isclose(trade.stop_loss, 1.76) # stoploss on exchange should not be canceled cancel_order_mock.assert_not_called() @@ -1575,10 +1575,12 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, # stoploss should be set to 1% as trailing is on assert trade.stop_loss == 4.4 * 0.99 cancel_order_mock.assert_called_once_with(100, 'NEO/BTC') - stoploss_order_mock.assert_called_once_with(amount=2132892.49146757, - pair='NEO/BTC', - order_types=freqtrade.strategy.order_types, - stop_price=4.4 * 0.99) + stoploss_order_mock.assert_called_once_with( + amount=11.41438356, + pair='NEO/BTC', + order_types=freqtrade.strategy.order_types, + stop_price=4.4 * 0.99 + ) @pytest.mark.parametrize('return_value,side_effect,log_message', [ From 6f8e66117bcc0babd0aeef9c03a15b9ee6fc2e11 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Thu, 30 Sep 2021 03:19:28 -0600 Subject: [PATCH 178/239] flake8 isort --- tests/test_freqtradebot.py | 153 ++++++++++++++++++++++--------------- 1 file changed, 91 insertions(+), 62 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index bfa3ac353..6f0fe5e90 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -23,9 +23,9 @@ from freqtrade.worker import Worker from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker, log_has, log_has_re, patch_edge, patch_exchange, patch_get_signal, patch_wallet, patch_whitelist) -from tests.conftest_trades import ( - MOCK_TRADE_COUNT, mock_order_1, mock_order_2, mock_order_2_sell, mock_order_3, - mock_order_3_sell, mock_order_4, mock_order_5_stoploss, mock_order_6_sell) +from tests.conftest_trades import (MOCK_TRADE_COUNT, mock_order_1, mock_order_2, mock_order_2_sell, + mock_order_3, mock_order_3_sell, mock_order_4, + mock_order_5_stoploss, mock_order_6_sell) def patch_RPCManager(mocker) -> MagicMock: @@ -135,8 +135,10 @@ def test_get_trade_stake_amount(default_conf_usdt, ticker_usdt, mocker) -> None: (True, 27, 3, 0.5, [10, 10, 6.73]), (True, 22, 3, 1, [10, 10, 0.0]), ]) -def test_check_available_stake_amount(default_conf_usdt, ticker_usdt, mocker, fee, limit_buy_order_usdt_open, - amend_last, wallet, max_open, lsamr, expected) -> None: +def test_check_available_stake_amount( + default_conf_usdt, ticker_usdt, mocker, fee, limit_buy_order_usdt_open, + amend_last, wallet, max_open, lsamr, expected +) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -363,8 +365,8 @@ def test_create_trade_minimal_amount( (["ETH/USDT"], 1), # No pairs left ([], 0), # No pairs in whitelist ]) -def test_enter_positions_no_pairs_left(default_conf_usdt, ticker_usdt, limit_buy_order_usdt_open, fee, - whitelist, positions, mocker, caplog) -> None: +def test_enter_positions_no_pairs_left(default_conf_usdt, ticker_usdt, limit_buy_order_usdt_open, + fee, whitelist, positions, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -459,8 +461,10 @@ def test_create_trade_no_signal(default_conf_usdt, fee, mocker) -> None: @pytest.mark.parametrize("max_open", range(0, 5)) @pytest.mark.parametrize("tradable_balance_ratio,modifier", [(1.0, 1), (0.99, 0.8), (0.5, 0.5)]) -def test_create_trades_multiple_trades(default_conf_usdt, ticker_usdt, fee, mocker, limit_buy_order_usdt_open, - max_open, tradable_balance_ratio, modifier) -> None: +def test_create_trades_multiple_trades( + default_conf_usdt, ticker_usdt, fee, mocker, limit_buy_order_usdt_open, + max_open, tradable_balance_ratio, modifier +) -> None: patch_RPCManager(mocker) patch_exchange(mocker) default_conf_usdt['max_open_trades'] = max_open @@ -484,7 +488,8 @@ def test_create_trades_multiple_trades(default_conf_usdt, ticker_usdt, fee, mock assert len(trades) == max(int(max_open * modifier), 0) -def test_create_trades_preopen(default_conf_usdt, ticker_usdt, fee, mocker, limit_buy_order_usdt_open) -> None: +def test_create_trades_preopen(default_conf_usdt, ticker_usdt, fee, mocker, + limit_buy_order_usdt_open) -> None: patch_RPCManager(mocker) patch_exchange(mocker) default_conf_usdt['max_open_trades'] = 4 @@ -513,8 +518,8 @@ def test_create_trades_preopen(default_conf_usdt, ticker_usdt, fee, mocker, limi assert len(trades) == 4 -def test_process_trade_creation(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, limit_buy_order_usdt_open, - fee, mocker, caplog) -> None: +def test_process_trade_creation(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, + limit_buy_order_usdt_open, fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -584,7 +589,8 @@ def test_process_operational_exception(default_conf_usdt, ticker_usdt, mocker) - assert 'OperationalException' in msg_mock.call_args_list[-1][0][0]['status'] -def test_process_trade_handling(default_conf_usdt, ticker_usdt, limit_buy_order_usdt_open, fee, mocker) -> None: +def test_process_trade_handling(default_conf_usdt, ticker_usdt, limit_buy_order_usdt_open, fee, + mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -690,7 +696,8 @@ def test_process_informative_pairs_added(default_conf_usdt, ticker_usdt, mocker) assert ("ETH/USDT", default_conf_usdt["timeframe"]) in refresh_mock.call_args[0][0] -def test_execute_entry(mocker, default_conf_usdt, fee, limit_buy_order_usdt, limit_buy_order_usdt_open) -> None: +def test_execute_entry(mocker, default_conf_usdt, fee, limit_buy_order_usdt, + limit_buy_order_usdt_open) -> None: patch_RPCManager(mocker) patch_exchange(mocker) freqtrade = FreqtradeBot(default_conf_usdt) @@ -1278,8 +1285,9 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf_usdt, fee, assert freqtrade.handle_trade(trade) is True -def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf_usdt, fee, caplog, - limit_buy_order_usdt, limit_sell_order_usdt) -> None: +def test_handle_stoploss_on_exchange_trailing_error( + mocker, default_conf_usdt, fee, caplog, limit_buy_order_usdt, limit_sell_order_usdt +) -> None: # When trailing stoploss is set stoploss = MagicMock(return_value={'id': 13434334}) patch_exchange(mocker) @@ -1701,8 +1709,8 @@ def test_update_trade_state(mocker, default_conf_usdt, limit_buy_order_usdt, cap (30.0 + 1e-14, True), (8.0, False) ]) -def test_update_trade_state_withorderdict(default_conf_usdt, trades_for_order, limit_buy_order_usdt, fee, - mocker, initial_amount, has_rounding_fee, caplog): +def test_update_trade_state_withorderdict(default_conf_usdt, trades_for_order, limit_buy_order_usdt, + fee, mocker, initial_amount, has_rounding_fee, caplog): trades_for_order[0]['amount'] = initial_amount mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) # fetch_order should not be called!! @@ -1798,8 +1806,8 @@ def test_update_trade_state_sell(default_conf_usdt, trades_for_order, limit_sell assert order.status == 'closed' -def test_handle_trade(default_conf_usdt, limit_buy_order_usdt, limit_sell_order_usdt_open, limit_sell_order_usdt, - fee, mocker) -> None: +def test_handle_trade(default_conf_usdt, limit_buy_order_usdt, limit_sell_order_usdt_open, + limit_sell_order_usdt, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -1963,8 +1971,8 @@ def test_handle_trade_use_sell_signal(default_conf_usdt, ticker_usdt, limit_buy_ caplog) -def test_close_trade(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, limit_buy_order_usdt_open, limit_sell_order_usdt, - fee, mocker) -> None: +def test_close_trade(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, + limit_buy_order_usdt_open, limit_sell_order_usdt, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -2003,8 +2011,8 @@ def test_bot_loop_start_called_once(mocker, default_conf_usdt, caplog): assert ftbot.strategy.analyze.call_count == 1 -def test_check_handle_timedout_buy_usercustom(default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, - fee, mocker) -> None: +def test_check_handle_timedout_buy_usercustom(default_conf_usdt, ticker_usdt, limit_buy_order_old, + open_trade, fee, mocker) -> None: default_conf_usdt["unfilledtimeout"] = {"buy": 1400, "sell": 30} rpc_mock = patch_RPCManager(mocker) @@ -2117,8 +2125,8 @@ def test_check_handle_cancelled_buy(default_conf_usdt, ticker_usdt, limit_buy_or assert log_has_re("Buy order cancelled on exchange for Trade.*", caplog) -def test_check_handle_timedout_buy_exception(default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, - fee, mocker) -> None: +def test_check_handle_timedout_buy_exception(default_conf_usdt, ticker_usdt, limit_buy_order_old, + open_trade, fee, mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() patch_exchange(mocker) @@ -2143,8 +2151,8 @@ def test_check_handle_timedout_buy_exception(default_conf_usdt, ticker_usdt, lim assert nb_trades == 1 -def test_check_handle_timedout_sell_usercustom(default_conf_usdt, ticker_usdt, limit_sell_order_old, mocker, - open_trade) -> None: +def test_check_handle_timedout_sell_usercustom(default_conf_usdt, ticker_usdt, limit_sell_order_old, + mocker, open_trade) -> None: default_conf_usdt["unfilledtimeout"] = {"buy": 1440, "sell": 1440} rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() @@ -2222,8 +2230,8 @@ def test_check_handle_timedout_sell(default_conf_usdt, ticker_usdt, limit_sell_o assert freqtrade.strategy.check_sell_timeout.call_count == 0 -def test_check_handle_cancelled_sell(default_conf_usdt, ticker_usdt, limit_sell_order_old, open_trade, - mocker, caplog) -> None: +def test_check_handle_cancelled_sell(default_conf_usdt, ticker_usdt, limit_sell_order_old, + open_trade, mocker, caplog) -> None: """ Handle sell order cancelled on exchange""" rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() @@ -2319,8 +2327,8 @@ def test_check_handle_timedout_partial_fee(default_conf_usdt, ticker_usdt, open_ assert pytest.approx(trades[0].fee_open) == 0.001 -def test_check_handle_timedout_partial_except(default_conf_usdt, ticker_usdt, open_trade, caplog, fee, - limit_buy_order_old_partial, trades_for_order, +def test_check_handle_timedout_partial_except(default_conf_usdt, ticker_usdt, open_trade, caplog, + fee, limit_buy_order_old_partial, trades_for_order, limit_buy_order_old_partial_canceled, mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock(return_value=limit_buy_order_old_partial_canceled) @@ -2359,7 +2367,8 @@ def test_check_handle_timedout_partial_except(default_conf_usdt, ticker_usdt, op assert trades[0].fee_open == fee() -def test_check_handle_timedout_exception(default_conf_usdt, ticker_usdt, open_trade_usdt, mocker, caplog) -> None: +def test_check_handle_timedout_exception(default_conf_usdt, ticker_usdt, open_trade_usdt, mocker, + caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock() @@ -2546,7 +2555,8 @@ def test_handle_cancel_exit_cancel_exception(mocker, default_conf_usdt) -> None: assert freqtrade.handle_cancel_exit(trade, order, reason) == 'error cancelling order' -def test_execute_trade_exit_up(default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_up, mocker) -> None: +def test_execute_trade_exit_up(default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_up, mocker + ) -> None: rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -2611,7 +2621,8 @@ def test_execute_trade_exit_up(default_conf_usdt, ticker_usdt, fee, ticker_usdt_ } == last_msg -def test_execute_trade_exit_down(default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_down, mocker) -> None: +def test_execute_trade_exit_down(default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_down, + mocker) -> None: rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -2664,8 +2675,8 @@ def test_execute_trade_exit_down(default_conf_usdt, ticker_usdt, fee, ticker_usd } == last_msg -def test_execute_trade_exit_custom_exit_price(default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_up, - mocker) -> None: +def test_execute_trade_exit_custom_exit_price(default_conf_usdt, ticker_usdt, fee, + ticker_usdt_sell_up, mocker) -> None: rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -2731,8 +2742,8 @@ def test_execute_trade_exit_custom_exit_price(default_conf_usdt, ticker_usdt, fe } == last_msg -def test_execute_trade_exit_down_stoploss_on_exchange_dry_run(default_conf_usdt, ticker_usdt, fee, - ticker_usdt_sell_down, mocker) -> None: +def test_execute_trade_exit_down_stoploss_on_exchange_dry_run( + default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_down, mocker) -> None: rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -2825,8 +2836,8 @@ def test_execute_trade_exit_sloe_cancel_exception( assert log_has('Could not cancel stoploss order abcd', caplog) -def test_execute_trade_exit_with_stoploss_on_exchange(default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_up, - mocker) -> None: +def test_execute_trade_exit_with_stoploss_on_exchange(default_conf_usdt, ticker_usdt, fee, + ticker_usdt_sell_up, mocker) -> None: default_conf_usdt['exchange']['name'] = 'binance' rpc_mock = patch_RPCManager(mocker) @@ -3169,7 +3180,8 @@ def test__safe_exit_amount(default_conf_usdt, fee, caplog, mocker, amount_wallet assert wallet_update.call_count == 1 -def test_locked_pairs(default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_down, mocker, caplog) -> None: +def test_locked_pairs(default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_down, mocker, + caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -3204,8 +3216,8 @@ def test_locked_pairs(default_conf_usdt, ticker_usdt, fee, ticker_usdt_sell_down assert log_has_re(f"Pair {trade.pair} is still locked.*", caplog) -def test_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usdt, limit_buy_order_usdt_open, - fee, mocker) -> None: +def test_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usdt, + limit_buy_order_usdt_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -3301,8 +3313,10 @@ def test_trailing_stop_loss(default_conf_usdt, limit_buy_order_usdt_open, limit_ (0.011, False, 2.0394), (0.055, True, 1.8), ]) -def test_trailing_stop_loss_positive(default_conf_usdt, limit_buy_order_usdt, limit_buy_order_usdt_open, - offset, fee, caplog, mocker, trail_if_reached, second_sl) -> None: +def test_trailing_stop_loss_positive( + default_conf_usdt, limit_buy_order_usdt, limit_buy_order_usdt_open, + offset, fee, caplog, mocker, trail_if_reached, second_sl +) -> None: buy_price = limit_buy_order_usdt['price'] patch_RPCManager(mocker) patch_exchange(mocker) @@ -3391,8 +3405,8 @@ def test_trailing_stop_loss_positive(default_conf_usdt, limit_buy_order_usdt, li assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value -def test_disable_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usdt, limit_buy_order_usdt_open, - fee, mocker) -> None: +def test_disable_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usdt, + limit_buy_order_usdt_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -3431,7 +3445,8 @@ def test_disable_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usd assert trade.sell_reason == SellType.SELL_SIGNAL.value -def test_get_real_amount_quote(default_conf_usdt, trades_for_order, buy_order_fee, fee, caplog, mocker): +def test_get_real_amount_quote(default_conf_usdt, trades_for_order, buy_order_fee, fee, caplog, + mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -3608,7 +3623,8 @@ def test_get_real_amount_multi( assert trade.fee_close_currency is None -def test_get_real_amount_invalid_order(default_conf_usdt, trades_for_order, buy_order_fee, fee, mocker): +def test_get_real_amount_invalid_order(default_conf_usdt, trades_for_order, buy_order_fee, fee, + mocker): limit_buy_order_usdt = deepcopy(buy_order_fee) limit_buy_order_usdt['fee'] = {'cost': 0.004} @@ -3629,7 +3645,8 @@ def test_get_real_amount_invalid_order(default_conf_usdt, trades_for_order, buy_ assert freqtrade.get_real_amount(trade, limit_buy_order_usdt) == amount -def test_get_real_amount_wrong_amount(default_conf_usdt, trades_for_order, buy_order_fee, fee, mocker): +def test_get_real_amount_wrong_amount(default_conf_usdt, trades_for_order, buy_order_fee, fee, + mocker): limit_buy_order_usdt = deepcopy(buy_order_fee) limit_buy_order_usdt['amount'] = limit_buy_order_usdt['amount'] - 0.001 @@ -3651,8 +3668,8 @@ def test_get_real_amount_wrong_amount(default_conf_usdt, trades_for_order, buy_o freqtrade.get_real_amount(trade, limit_buy_order_usdt) -def test_get_real_amount_wrong_amount_rounding(default_conf_usdt, trades_for_order, buy_order_fee, fee, - mocker): +def test_get_real_amount_wrong_amount_rounding(default_conf_usdt, trades_for_order, buy_order_fee, + fee, mocker): # Floats should not be compared directly. limit_buy_order_usdt = deepcopy(buy_order_fee) trades_for_order[0]['amount'] = trades_for_order[0]['amount'] + 1e-15 @@ -3671,8 +3688,11 @@ def test_get_real_amount_wrong_amount_rounding(default_conf_usdt, trades_for_ord freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) # Amount changes by fee amount. - assert isclose(freqtrade.get_real_amount(trade, limit_buy_order_usdt), amount - (amount * 0.001), - abs_tol=MATH_CLOSE_PREC,) + assert isclose( + freqtrade.get_real_amount(trade, limit_buy_order_usdt), + amount - (amount * 0.001), + abs_tol=MATH_CLOSE_PREC, + ) def test_get_real_amount_open_trade(default_conf_usdt, fee, mocker): @@ -3729,8 +3749,10 @@ def test_apply_fee_conditional(default_conf_usdt, fee, caplog, mocker, (0.1, False), (100, True), ]) -def test_order_book_depth_of_market(default_conf_usdt, ticker_usdt, limit_buy_order_usdt_open, limit_buy_order_usdt, - fee, mocker, order_book_l2, delta, is_high_delta): +def test_order_book_depth_of_market( + default_conf_usdt, ticker_usdt, limit_buy_order_usdt_open, limit_buy_order_usdt, + fee, mocker, order_book_l2, delta, is_high_delta +): default_conf_usdt['bid_strategy']['check_depth_of_market']['enabled'] = True default_conf_usdt['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = delta patch_RPCManager(mocker) @@ -3821,8 +3843,9 @@ def test_check_depth_of_market_buy(default_conf_usdt, mocker, order_book_l2) -> assert freqtrade._check_depth_of_market_buy('ETH/USDT', conf) is False -def test_order_book_ask_strategy(default_conf_usdt, limit_buy_order_usdt_open, limit_buy_order_usdt, fee, - limit_sell_order_usdt_open, mocker, order_book_l2, caplog) -> None: +def test_order_book_ask_strategy( + default_conf_usdt, limit_buy_order_usdt_open, limit_buy_order_usdt, fee, + limit_sell_order_usdt_open, mocker, order_book_l2, caplog) -> None: """ test order book ask strategy """ @@ -3898,7 +3921,8 @@ def test_startup_trade_reinit(default_conf_usdt, edge_conf, mocker): @pytest.mark.usefixtures("init_persistence") -def test_sync_wallet_dry_run(mocker, default_conf_usdt, ticker_usdt, fee, limit_buy_order_usdt_open, caplog): +def test_sync_wallet_dry_run(mocker, default_conf_usdt, ticker_usdt, fee, limit_buy_order_usdt_open, + caplog): default_conf_usdt['dry_run'] = True # Initialize to 2 times stake amount default_conf_usdt['dry_run_wallet'] = 20.0 @@ -3930,11 +3954,16 @@ def test_sync_wallet_dry_run(mocker, default_conf_usdt, ticker_usdt, fee, limit_ @pytest.mark.usefixtures("init_persistence") -def test_cancel_all_open_orders(mocker, default_conf_usdt, fee, limit_buy_order_usdt, limit_sell_order_usdt): +def test_cancel_all_open_orders(mocker, default_conf_usdt, fee, limit_buy_order_usdt, + limit_sell_order_usdt): default_conf_usdt['cancel_open_orders_on_exit'] = True mocker.patch('freqtrade.exchange.Exchange.fetch_order', side_effect=[ - ExchangeError(), limit_sell_order_usdt, limit_buy_order_usdt, limit_sell_order_usdt]) + ExchangeError(), + limit_sell_order_usdt, + limit_buy_order_usdt, + limit_sell_order_usdt + ]) buy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_enter') sell_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_exit') From 96d09b5615c14fd442394060d844be2c898056a6 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 1 Oct 2021 03:10:43 -0600 Subject: [PATCH 179/239] Fixed breaking rpc tests --- tests/rpc/test_rpc.py | 2 +- tests/rpc/test_rpc_apiserver.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 0ba42c4ce..f8c923958 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -1003,7 +1003,7 @@ def test_rpc_blacklist(mocker, default_conf) -> None: assert len(ret['blacklist']) == 4 assert ret['blacklist'] == default_conf['exchange']['pair_blacklist'] assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC', 'ETH/BTC', 'XRP/.*'] - assert ret['blacklist_expanded'] == ['ETH/BTC', 'XRP/BTC'] + assert ret['blacklist_expanded'] == ['ETH/BTC', 'XRP/BTC', 'XRP/USDT'] assert 'errors' in ret assert isinstance(ret['errors'], dict) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 7c98b2df7..045f91bb8 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -937,7 +937,7 @@ def test_api_blacklist(botclient, mocker): data='{"blacklist": ["XRP/.*"]}') assert_response(rc) assert rc.json() == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC", "XRP/.*"], - "blacklist_expanded": ["ETH/BTC", "XRP/BTC"], + "blacklist_expanded": ["ETH/BTC", "XRP/BTC", "XRP/USDT"], "length": 4, "method": ["StaticPairList"], "errors": {}, From e5e1e49f5327753ed9e7525c5cdca838e2c3ab66 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Oct 2021 08:58:33 +0200 Subject: [PATCH 180/239] Remove some unused test parameters --- tests/test_freqtradebot.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 6f0fe5e90..02c913bc3 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -115,7 +115,7 @@ def test_order_dict(default_conf_usdt, mocker, runmode, caplog) -> None: assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) -def test_get_trade_stake_amount(default_conf_usdt, ticker_usdt, mocker) -> None: +def test_get_trade_stake_amount(default_conf_usdt, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -243,7 +243,6 @@ def test_edge_overrides_stoploss(limit_buy_order_usdt, fee, caplog, mocker, def test_total_open_trades_stakes(mocker, default_conf_usdt, ticker_usdt, fee) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - default_conf_usdt['stake_amount'] = 10.0 default_conf_usdt['max_open_trades'] = 2 mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -304,8 +303,7 @@ def test_create_trade(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, fee, assert whitelist == default_conf_usdt['exchange']['pair_whitelist'] -def test_create_trade_no_stake_amount(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, - fee, mocker) -> None: +def test_create_trade_no_stake_amount(default_conf_usdt, ticker_usdt, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) patch_wallet(mocker, free=default_conf_usdt['stake_amount'] * 0.5) @@ -1477,7 +1475,7 @@ def test_handle_stoploss_on_exchange_custom_stop( assert freqtrade.handle_trade(trade) is True -def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, +def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, limit_buy_order_usdt, limit_sell_order_usdt) -> None: # When trailing stoploss is set @@ -2125,7 +2123,7 @@ def test_check_handle_cancelled_buy(default_conf_usdt, ticker_usdt, limit_buy_or assert log_has_re("Buy order cancelled on exchange for Trade.*", caplog) -def test_check_handle_timedout_buy_exception(default_conf_usdt, ticker_usdt, limit_buy_order_old, +def test_check_handle_timedout_buy_exception(default_conf_usdt, ticker_usdt, open_trade, fee, mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() @@ -3253,7 +3251,7 @@ def test_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usdt, assert trade.sell_reason == SellType.ROI.value -def test_trailing_stop_loss(default_conf_usdt, limit_buy_order_usdt_open, limit_buy_order_usdt, +def test_trailing_stop_loss(default_conf_usdt, limit_buy_order_usdt_open, fee, caplog, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -3724,7 +3722,7 @@ def test_get_real_amount_open_trade(default_conf_usdt, fee, mocker): (8.0, 0.1, 8.0, 8.0), (8.0, 0.1, 7.9, 7.9), ]) -def test_apply_fee_conditional(default_conf_usdt, fee, caplog, mocker, +def test_apply_fee_conditional(default_conf_usdt, fee, mocker, amount, fee_abs, wallet, amount_exp): walletmock = mocker.patch('freqtrade.wallets.Wallets.update') mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=wallet) From 022839b728fafdf2f1a83d514fb0c9acd027db54 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Oct 2021 13:17:10 +0200 Subject: [PATCH 181/239] remove unnecessary test --- tests/test_freqtradebot.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 02c913bc3..c169d8597 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1612,32 +1612,6 @@ def test_enter_positions(mocker, default_conf_usdt, return_value, side_effect, assert mock_ct.call_count == len(default_conf_usdt['exchange']['pair_whitelist']) -def test_exit_positions(mocker, default_conf_usdt, limit_buy_order_usdt, caplog) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) - - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) - mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order_usdt) - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', - return_value=limit_buy_order_usdt['amount']) - - trade = MagicMock() - trade.open_order_id = '123' - trade.open_fee = 0.001 - trades = [trade] - n = freqtrade.exit_positions(trades) - assert n == 0 - # Test amount not modified by fee-logic - assert not log_has( - 'Applying fee to amount for Trade {} from 30.0 to 90.81'.format(trade), caplog - ) - - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=90.81) - # test amount modified by fee-logic - n = freqtrade.exit_positions(trades) - assert n == 0 - - def test_exit_positions_exception(mocker, default_conf_usdt, limit_buy_order_usdt, caplog) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order_usdt) From 5fdeca812d28891a915507362ffcc9f36ccb4cb0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Oct 2021 14:30:24 +0200 Subject: [PATCH 182/239] Combine most hyperopt-loss tests to one --- tests/optimize/conftest.py | 13 ++-- tests/optimize/test_hyperoptloss.py | 92 +++++------------------------ 2 files changed, 23 insertions(+), 82 deletions(-) diff --git a/tests/optimize/conftest.py b/tests/optimize/conftest.py index 5c5171c3a..690934b07 100644 --- a/tests/optimize/conftest.py +++ b/tests/optimize/conftest.py @@ -39,16 +39,17 @@ def hyperopt(hyperopt_conf, mocker): def hyperopt_results(): return pd.DataFrame( { - 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], - 'profit_ratio': [-0.1, 0.2, 0.3], - 'profit_abs': [-0.2, 0.4, 0.6], - 'trade_duration': [10, 30, 10], - 'sell_reason': [SellType.STOP_LOSS, SellType.ROI, SellType.ROI], + 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC', 'ETH/BTC'], + 'profit_ratio': [-0.1, 0.2, -0.1, 0.3], + 'profit_abs': [-0.2, 0.4, -0.2, 0.6], + 'trade_duration': [10, 30, 10, 10], + 'sell_reason': [SellType.STOP_LOSS, SellType.ROI, SellType.STOP_LOSS, SellType.ROI], 'close_date': [ datetime(2019, 1, 1, 9, 26, 3, 478039), datetime(2019, 2, 1, 9, 26, 3, 478039), - datetime(2019, 3, 1, 9, 26, 3, 478039) + datetime(2019, 3, 1, 9, 26, 3, 478039), + datetime(2019, 4, 1, 9, 26, 3, 478039), ] } ) diff --git a/tests/optimize/test_hyperoptloss.py b/tests/optimize/test_hyperoptloss.py index 0082bcc34..923e3fc32 100644 --- a/tests/optimize/test_hyperoptloss.py +++ b/tests/optimize/test_hyperoptloss.py @@ -35,6 +35,7 @@ def test_hyperoptlossresolver_wrongname(default_conf) -> None: def test_loss_calculation_prefer_correct_trade_count(hyperopt_conf, hyperopt_results) -> None: + hyperopt_conf.update({'hyperopt_loss': "ShortTradeDurHyperOptLoss"}) hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) correct = hl.hyperopt_loss_function(hyperopt_results, 600, datetime(2019, 1, 1), datetime(2019, 5, 1)) @@ -50,6 +51,7 @@ def test_loss_calculation_prefer_shorter_trades(hyperopt_conf, hyperopt_results) resultsb = hyperopt_results.copy() resultsb.loc[1, 'trade_duration'] = 20 + hyperopt_conf.update({'hyperopt_loss': "ShortTradeDurHyperOptLoss"}) hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) longer = hl.hyperopt_loss_function(hyperopt_results, 100, datetime(2019, 1, 1), datetime(2019, 5, 1)) @@ -64,6 +66,7 @@ def test_loss_calculation_has_limited_profit(hyperopt_conf, hyperopt_results) -> results_under = hyperopt_results.copy() results_under['profit_ratio'] = hyperopt_results['profit_ratio'] / 2 + hyperopt_conf.update({'hyperopt_loss': "ShortTradeDurHyperOptLoss"}) hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) correct = hl.hyperopt_loss_function(hyperopt_results, 600, datetime(2019, 1, 1), datetime(2019, 5, 1)) @@ -75,91 +78,28 @@ def test_loss_calculation_has_limited_profit(hyperopt_conf, hyperopt_results) -> assert under > correct -def test_sharpe_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_ratio'] = hyperopt_results['profit_ratio'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_ratio'] = hyperopt_results['profit_ratio'] / 2 - - default_conf.update({'hyperopt_loss': 'SharpeHyperOptLoss'}) - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - -def test_sharpe_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_ratio'] = hyperopt_results['profit_ratio'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_ratio'] = hyperopt_results['profit_ratio'] / 2 - - default_conf.update({'hyperopt_loss': 'SharpeHyperOptLossDaily'}) - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - -def test_sortino_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_ratio'] = hyperopt_results['profit_ratio'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_ratio'] = hyperopt_results['profit_ratio'] / 2 - - default_conf.update({'hyperopt_loss': 'SortinoHyperOptLoss'}) - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - -def test_sortino_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_ratio'] = hyperopt_results['profit_ratio'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_ratio'] = hyperopt_results['profit_ratio'] / 2 - - default_conf.update({'hyperopt_loss': 'SortinoHyperOptLossDaily'}) - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - -def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: +@pytest.mark.parametrize('lossfunction', [ + "OnlyProfitHyperOptLoss", + "SortinoHyperOptLoss", + "SortinoHyperOptLossDaily", + "SharpeHyperOptLoss", + "SharpeHyperOptLossDaily", +]) +def test_loss_functions_better_profits(default_conf, hyperopt_results, lossfunction) -> None: results_over = hyperopt_results.copy() results_over['profit_abs'] = hyperopt_results['profit_abs'] * 2 + results_over['profit_ratio'] = hyperopt_results['profit_ratio'] * 2 results_under = hyperopt_results.copy() results_under['profit_abs'] = hyperopt_results['profit_abs'] / 2 + results_under['profit_ratio'] = hyperopt_results['profit_ratio'] / 2 - default_conf.update({'hyperopt_loss': 'OnlyProfitHyperOptLoss'}) + default_conf.update({'hyperopt_loss': lossfunction}) hl = HyperOptLossResolver.load_hyperoptloss(default_conf) correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + over = hl.hyperopt_loss_function(results_over, len(results_over), datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + under = hl.hyperopt_loss_function(results_under, len(results_under), datetime(2019, 1, 1), datetime(2019, 5, 1)) assert over < correct assert under > correct From 77388eb423f8867aa63860ace494dd85505dd416 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Oct 2021 15:23:48 +0200 Subject: [PATCH 183/239] Improve generate_test_data to make it easier to use --- tests/optimize/conftest.py | 20 ++++++++++++++------ tests/strategy/test_strategy_helpers.py | 4 ++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/optimize/conftest.py b/tests/optimize/conftest.py index 690934b07..13d90b36f 100644 --- a/tests/optimize/conftest.py +++ b/tests/optimize/conftest.py @@ -39,17 +39,25 @@ def hyperopt(hyperopt_conf, mocker): def hyperopt_results(): return pd.DataFrame( { - 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC', 'ETH/BTC'], + 'pair': ['ETH/USDT', 'ETH/USDT', 'ETH/USDT', 'ETH/USDT'], 'profit_ratio': [-0.1, 0.2, -0.1, 0.3], 'profit_abs': [-0.2, 0.4, -0.2, 0.6], 'trade_duration': [10, 30, 10, 10], + 'amount': [0.1, 0.1, 0.1, 0.1], 'sell_reason': [SellType.STOP_LOSS, SellType.ROI, SellType.STOP_LOSS, SellType.ROI], + 'open_date': + [ + datetime(2019, 1, 1, 9, 15, 0), + datetime(2019, 2, 1, 8, 55, 0), + datetime(2019, 3, 1, 9, 15, 0), + datetime(2019, 4, 1, 9, 15, 0), + ], 'close_date': [ - datetime(2019, 1, 1, 9, 26, 3, 478039), - datetime(2019, 2, 1, 9, 26, 3, 478039), - datetime(2019, 3, 1, 9, 26, 3, 478039), - datetime(2019, 4, 1, 9, 26, 3, 478039), - ] + datetime(2019, 1, 1, 9, 25, 0), + datetime(2019, 2, 1, 9, 25, 0), + datetime(2019, 3, 1, 9, 25, 0), + datetime(2019, 4, 1, 9, 25, 0), + ], } ) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index a01b55050..cb7cf97a1 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -9,13 +9,13 @@ from freqtrade.strategy import (merge_informative_pair, stoploss_from_absolute, timeframe_to_minutes) -def generate_test_data(timeframe: str, size: int): +def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'): np.random.seed(42) tf_mins = timeframe_to_minutes(timeframe) base = np.random.normal(20, 2, size=size) - date = pd.period_range('2020-07-05', periods=size, freq=f'{tf_mins}min').to_timestamp() + date = pd.date_range(start, periods=size, freq=f'{tf_mins}min', tz='UTC') df = pd.DataFrame({ 'date': date, 'open': base, From 3b5cc5f01584329b3d9b0bd47cc71dbf97e8e3d1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Oct 2021 15:36:51 +0200 Subject: [PATCH 184/239] Improve dates used for hyperopt tests --- tests/optimize/conftest.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/optimize/conftest.py b/tests/optimize/conftest.py index 13d90b36f..8c7fa3ac9 100644 --- a/tests/optimize/conftest.py +++ b/tests/optimize/conftest.py @@ -48,16 +48,16 @@ def hyperopt_results(): 'open_date': [ datetime(2019, 1, 1, 9, 15, 0), - datetime(2019, 2, 1, 8, 55, 0), - datetime(2019, 3, 1, 9, 15, 0), - datetime(2019, 4, 1, 9, 15, 0), + datetime(2019, 1, 2, 8, 55, 0), + datetime(2019, 1, 3, 9, 15, 0), + datetime(2019, 1, 4, 9, 15, 0), ], 'close_date': [ datetime(2019, 1, 1, 9, 25, 0), - datetime(2019, 2, 1, 9, 25, 0), - datetime(2019, 3, 1, 9, 25, 0), - datetime(2019, 4, 1, 9, 25, 0), + datetime(2019, 1, 2, 9, 25, 0), + datetime(2019, 1, 3, 9, 25, 0), + datetime(2019, 1, 4, 9, 25, 0), ], } ) From 057a187231bf5d90d0d2494d66934593d01133e8 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 2 Oct 2021 20:32:51 -0600 Subject: [PATCH 185/239] Removed uneccessary TODOs --- tests/test_freqtradebot.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index c169d8597..f80ffeba2 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2581,7 +2581,6 @@ def test_execute_trade_exit_up(default_conf_usdt, ticker_usdt, fee, ticker_usdt_ 'order_type': 'limit', 'open_rate': 2.0, 'current_rate': 2.3, - # TODO: Double check that profit_amount and profit_ratio are correct 'profit_amount': 0.9475, 'profit_ratio': 0.09451372, 'stake_currency': 'USDT', @@ -2624,7 +2623,6 @@ def test_execute_trade_exit_down(default_conf_usdt, ticker_usdt, fee, ticker_usd assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] - # TODO: Should be a loss, but comes out as a gain assert { 'type': RPCMessageType.SELL, 'trade_id': 1, @@ -2751,7 +2749,6 @@ def test_execute_trade_exit_down_stoploss_on_exchange_dry_run( assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] - # TODO: Are these values correct? assert { 'type': RPCMessageType.SELL, 'trade_id': 1, @@ -3274,7 +3271,6 @@ def test_trailing_stop_loss(default_conf_usdt, limit_buy_order_usdt_open, caplog.set_level(logging.DEBUG) # Sell as trailing-stop is reached assert freqtrade.handle_trade(trade) is True - # TODO: Does this make sense? How is stoploss 2.7? assert log_has("ETH/USDT - HIT STOP: current price at 2.200000, stoploss is 2.700000, " "initial stoploss was at 1.800000, trade opened at 2.000000", caplog) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value From 93679db7c4bc63d9c6541d90e36dd361e6b26568 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 2 Oct 2021 20:33:46 -0600 Subject: [PATCH 186/239] Removed ... TODOs --- tests/test_freqtradebot.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index f80ffeba2..97f8e92c8 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2962,11 +2962,10 @@ def test_execute_trade_exit_market_order(default_conf_usdt, ticker_usdt, fee, sell_reason=SellCheckTuple(sell_type=SellType.ROI)) assert not trade.is_open - assert trade.close_profit == 0.09451372 # TODO: Check this is correct + assert trade.close_profit == 0.09451372 assert rpc_mock.call_count == 3 last_msg = rpc_mock.call_args_list[-1][0][0] - # TODO: Is this correct? assert { 'type': RPCMessageType.SELL, 'trade_id': 1, @@ -3330,7 +3329,6 @@ def test_trailing_stop_loss_positive( ) # stop-loss not reached, adjusted stoploss assert freqtrade.handle_trade(trade) is False - # TODO: is 0.0249% correct? Shouldn't it be higher? caplog_text = f"ETH/USDT - Using positive stoploss: 0.01 offset: {offset} profit: 0.0249%" if trail_if_reached: assert not log_has(caplog_text, caplog) From 908dee961d091b4ec9bb299d658011a557540c69 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 2 Oct 2021 20:37:05 -0600 Subject: [PATCH 187/239] Changed test values in test_sell_profit_only to usdt like values --- tests/test_freqtradebot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 97f8e92c8..3f14c75c4 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3026,12 +3026,12 @@ def test_execute_trade_exit_insufficient_funds_error(default_conf_usdt, ticker_u # Enable profit (True, 1.9, 2.2, False, True, SellType.SELL_SIGNAL.value), # Disable profit - (False, 0.00002172, 0.00002173, True, False, SellType.SELL_SIGNAL.value), + (False, 2.9, 3.2, True, False, SellType.SELL_SIGNAL.value), # Enable loss # * Shouldn't this be SellType.STOP_LOSS.value - (True, 0.00000172, 0.00000173, False, False, None), + (True, 0.19, 0.22, False, False, None), # Disable loss - (False, 0.00000172, 0.00000173, True, False, SellType.SELL_SIGNAL.value), + (False, 0.10, 0.22, True, False, SellType.SELL_SIGNAL.value), ]) def test_sell_profit_only( default_conf_usdt, limit_buy_order_usdt, limit_buy_order_usdt_open, From 058c7b3e992971bf53f752309ff0dcc91cefe542 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 2 Oct 2021 20:43:32 -0600 Subject: [PATCH 188/239] Fixed odd test_execute_entry where the filled coins were higher than the amount --- tests/test_freqtradebot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 3f14c75c4..b518c02ad 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -773,10 +773,10 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_buy_order_usdt, # In case of rejected or expired order and partially filled limit_buy_order_usdt['status'] = 'expired' limit_buy_order_usdt['amount'] = 30.0 - limit_buy_order_usdt['filled'] = 80.99181073 + limit_buy_order_usdt['filled'] = 20.0 limit_buy_order_usdt['remaining'] = 10.00 limit_buy_order_usdt['price'] = 0.5 - limit_buy_order_usdt['cost'] = 40.495905365 + limit_buy_order_usdt['cost'] = 15.0 limit_buy_order_usdt['id'] = '555' mocker.patch('freqtrade.exchange.Exchange.create_order', MagicMock(return_value=limit_buy_order_usdt)) @@ -785,7 +785,7 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_buy_order_usdt, assert trade assert trade.open_order_id == '555' assert trade.open_rate == 0.5 - assert trade.stake_amount == 40.495905365 + assert trade.stake_amount == 15.0 # Test with custom stake limit_buy_order_usdt['status'] = 'open' From 9e77a739fa18218de4d5efc89bd910a5ac7cdc02 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Oct 2021 09:22:50 +0200 Subject: [PATCH 189/239] Change usdt stake_amount to 60$ --- tests/conftest.py | 2 +- tests/test_freqtradebot.py | 62 +++++++++++++++++++------------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0e5e7c933..49534c88d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -376,7 +376,7 @@ def get_default_conf(testdatadir): def get_default_conf_usdt(testdatadir): configuration = get_default_conf(testdatadir) configuration.update({ - "stake_amount": 10.0, + "stake_amount": 60.0, "stake_currency": "USDT", "exchange": { "name": "binance", diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index b518c02ad..f57e35ca1 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -126,14 +126,14 @@ def test_get_trade_stake_amount(default_conf_usdt, mocker) -> None: @pytest.mark.parametrize("amend_last,wallet,max_open,lsamr,expected", [ - (False, 20, 2, 0.5, [10, None]), - (True, 20, 2, 0.5, [10, 9.8]), - (False, 30, 3, 0.5, [10, 10, None]), - (True, 30, 3, 0.5, [10, 10, 9.7]), - (False, 22, 3, 0.5, [10, 10, None]), - (True, 22, 3, 0.5, [10, 10, 0.0]), - (True, 27, 3, 0.5, [10, 10, 6.73]), - (True, 22, 3, 1, [10, 10, 0.0]), + (False, 120, 2, 0.5, [60, None]), + (True, 120, 2, 0.5, [60, 58.8]), + (False, 180, 3, 0.5, [60, 60, None]), + (True, 180, 3, 0.5, [60, 60, 58.2]), + (False, 122, 3, 0.5, [60, 60, None]), + (True, 122, 3, 0.5, [60, 60, 0.0]), + (True, 167, 3, 0.5, [60, 60, 45.33]), + (True, 122, 3, 1, [60, 60, 0.0]), ]) def test_check_available_stake_amount( default_conf_usdt, ticker_usdt, mocker, fee, limit_buy_order_usdt_open, @@ -256,7 +256,7 @@ def test_total_open_trades_stakes(mocker, default_conf_usdt, ticker_usdt, fee) - trade = Trade.query.first() assert trade is not None - assert trade.stake_amount == 10.0 + assert trade.stake_amount == 60.0 assert trade.is_open assert trade.open_date is not None @@ -264,11 +264,11 @@ def test_total_open_trades_stakes(mocker, default_conf_usdt, ticker_usdt, fee) - trade = Trade.query.order_by(Trade.id.desc()).first() assert trade is not None - assert trade.stake_amount == 10.0 + assert trade.stake_amount == 60.0 assert trade.is_open assert trade.open_date is not None - assert Trade.total_open_trades_stakes() == 20.0 + assert Trade.total_open_trades_stakes() == 120.0 def test_create_trade(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, fee, mocker) -> None: @@ -289,7 +289,7 @@ def test_create_trade(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, fee, trade = Trade.query.first() assert trade is not None - assert trade.stake_amount == 10.0 + assert trade.stake_amount == 60.0 assert trade.is_open assert trade.open_date is not None assert trade.exchange == 'binance' @@ -467,7 +467,7 @@ def test_create_trades_multiple_trades( patch_exchange(mocker) default_conf_usdt['max_open_trades'] = max_open default_conf_usdt['tradable_balance_ratio'] = tradable_balance_ratio - default_conf_usdt['dry_run_wallet'] = 10.0 * max_open + default_conf_usdt['dry_run_wallet'] = 60.0 * max_open mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -544,10 +544,10 @@ def test_process_trade_creation(default_conf_usdt, ticker_usdt, limit_buy_order_ assert trade.open_date is not None assert trade.exchange == 'binance' assert trade.open_rate == 2.0 - assert trade.amount == 5.0 + assert trade.amount == 30.0 assert log_has( - 'Buy signal found: about create a new trade for ETH/USDT with stake_amount: 10.0 ...', + 'Buy signal found: about create a new trade for ETH/USDT with stake_amount: 60.0 ...', caplog ) @@ -1265,7 +1265,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf_usdt, fee, cancel_order_mock.assert_called_once_with(100, 'ETH/USDT') stoploss_order_mock.assert_called_once_with( - amount=4.56621004, + amount=27.39726027, pair='ETH/USDT', order_types=freqtrade.strategy.order_types, stop_price=4.4 * 0.95 @@ -1457,7 +1457,7 @@ def test_handle_stoploss_on_exchange_custom_stop( cancel_order_mock.assert_called_once_with(100, 'ETH/USDT') stoploss_order_mock.assert_called_once_with( - amount=5.26315789, + amount=31.57894736, pair='ETH/USDT', order_types=freqtrade.strategy.order_types, stop_price=4.4 * 0.96 @@ -2577,11 +2577,11 @@ def test_execute_trade_exit_up(default_conf_usdt, ticker_usdt, fee, ticker_usdt_ 'pair': 'ETH/USDT', 'gain': 'profit', 'limit': 2.2, - 'amount': 5.0, + 'amount': 30.0, 'order_type': 'limit', 'open_rate': 2.0, 'current_rate': 2.3, - 'profit_amount': 0.9475, + 'profit_amount': 5.685, 'profit_ratio': 0.09451372, 'stake_currency': 'USDT', 'fiat_currency': 'USD', @@ -2630,11 +2630,11 @@ def test_execute_trade_exit_down(default_conf_usdt, ticker_usdt, fee, ticker_usd 'pair': 'ETH/USDT', 'gain': 'loss', 'limit': 2.01, - 'amount': 5.0, + 'amount': 30.0, 'order_type': 'limit', 'open_rate': 2.0, 'current_rate': 2.0, - 'profit_amount': -0.000125, + 'profit_amount': -0.00075, 'profit_ratio': -1.247e-05, 'stake_currency': 'USDT', 'fiat_currency': 'USD', @@ -2697,11 +2697,11 @@ def test_execute_trade_exit_custom_exit_price(default_conf_usdt, ticker_usdt, fe 'pair': 'ETH/USDT', 'gain': 'profit', 'limit': 2.25, - 'amount': 5.0, + 'amount': 30.0, 'order_type': 'limit', 'open_rate': 2.0, 'current_rate': 2.3, - 'profit_amount': 1.196875, + 'profit_amount': 7.18125, 'profit_ratio': 0.11938903, 'stake_currency': 'USDT', 'fiat_currency': 'USD', @@ -2756,11 +2756,11 @@ def test_execute_trade_exit_down_stoploss_on_exchange_dry_run( 'pair': 'ETH/USDT', 'gain': 'loss', 'limit': 1.98, - 'amount': 5.0, + 'amount': 30.0, 'order_type': 'limit', 'open_rate': 2.0, 'current_rate': 2.0, - 'profit_amount': -0.14975, + 'profit_amount': -0.8985, 'profit_ratio': -0.01493766, 'stake_currency': 'USDT', 'fiat_currency': 'USD', @@ -2973,11 +2973,11 @@ def test_execute_trade_exit_market_order(default_conf_usdt, ticker_usdt, fee, 'pair': 'ETH/USDT', 'gain': 'profit', 'limit': 2.2, - 'amount': 5.0, + 'amount': 30.0, 'order_type': 'market', 'open_rate': 2.0, 'current_rate': 2.3, - 'profit_amount': 0.9475, + 'profit_amount': 5.685, 'profit_ratio': 0.09451372, 'stake_currency': 'USDT', 'fiat_currency': 'USD', @@ -3742,7 +3742,7 @@ def test_order_book_depth_of_market( assert trade is None else: assert trade is not None - assert trade.stake_amount == 10.0 + assert trade.stake_amount == 60.0 assert trade.is_open assert trade.open_date is not None assert trade.exchange == 'binance' @@ -3891,7 +3891,7 @@ def test_sync_wallet_dry_run(mocker, default_conf_usdt, ticker_usdt, fee, limit_ caplog): default_conf_usdt['dry_run'] = True # Initialize to 2 times stake amount - default_conf_usdt['dry_run_wallet'] = 20.0 + default_conf_usdt['dry_run_wallet'] = 120.0 default_conf_usdt['max_open_trades'] = 2 default_conf_usdt['tradable_balance_ratio'] = 1.0 patch_exchange(mocker) @@ -3904,7 +3904,7 @@ def test_sync_wallet_dry_run(mocker, default_conf_usdt, ticker_usdt, fee, limit_ bot = get_patched_freqtradebot(mocker, default_conf_usdt) patch_get_signal(bot) - assert bot.wallets.get_free('USDT') == 20.0 + assert bot.wallets.get_free('USDT') == 120.0 n = bot.enter_positions() assert n == 2 @@ -3915,7 +3915,7 @@ def test_sync_wallet_dry_run(mocker, default_conf_usdt, ticker_usdt, fee, limit_ n = bot.enter_positions() assert n == 0 assert log_has_re(r"Unable to create trade for XRP/USDT: " - r"Available balance \(0.0 USDT\) is lower than stake amount \(10.0 USDT\)", + r"Available balance \(0.0 USDT\) is lower than stake amount \(60.0 USDT\)", caplog) From 126c29198888508c563536a8d5e6221d2bcb36e9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Oct 2021 09:32:53 +0200 Subject: [PATCH 190/239] Improve docs closes #5654 --- docs/docker_quickstart.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 2f350d207..27a9091b1 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -109,6 +109,7 @@ All freqtrade arguments will be available by running `docker-compose run --rm fr !!! Warning "`docker-compose` for trade commands" Trade commands (`freqtrade trade <...>`) should not be ran via `docker-compose run` - but should use `docker-compose up -d` instead. This makes sure that the container is properly started (including port forwardings) and will make sure that the container will restart after a system reboot. + If you intend to use freqUI, please also ensure to adjust the [configuration accordingly](rest-api.md#configuration-with-docker), otherwise the UI will not be available. !!! Note "`docker-compose run --rm`" Including `--rm` will remove the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command). From e73f5ab4802823b6cfa2dba745d6218be0a7f97b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Oct 2021 09:48:19 +0200 Subject: [PATCH 191/239] Add test confirming #5652 --- tests/exchange/test_exchange.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 79b4a3ff5..691cf3c03 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -275,6 +275,7 @@ def test_amount_to_precision(default_conf, mocker, amount, precision_mode, preci (234.43, 4, 0.5, 234.5), (234.53, 4, 0.5, 235.0), (0.891534, 4, 0.0001, 0.8916), + (64968.89, 4, 0.01, 64968.89), ]) def test_price_to_precision(default_conf, mocker, price, precision_mode, precision, expected): @@ -293,7 +294,7 @@ def test_price_to_precision(default_conf, mocker, price, precision_mode, precisi PropertyMock(return_value=precision_mode)) pair = 'ETH/BTC' - assert pytest.approx(exchange.price_to_precision(pair, price)) == expected + assert exchange.price_to_precision(pair, price) == expected @pytest.mark.parametrize("price,precision_mode,precision,expected", [ From f5e5203388b6666664c5b9edd09683956d196478 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Oct 2021 09:48:50 +0200 Subject: [PATCH 192/239] Use "round" to 12 digits for TickSize mode Avoids float rounding problems, fix #5652 --- freqtrade/exchange/exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 2b9b08d70..e9d0316d2 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -523,7 +523,7 @@ class Exchange: precision = self.markets[pair]['precision']['price'] missing = price % precision if missing != 0: - price = price - missing + precision + price = round(price - missing + precision, 10) else: symbol_prec = self.markets[pair]['precision']['price'] big_price = price * pow(10, symbol_prec) From 1c63d01cec878e363075e2f43d79296be2ab3d60 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Oct 2021 14:14:16 +0200 Subject: [PATCH 193/239] Prevent using market-orders on gateio GateIo does not support market orders on spot markets --- freqtrade/exchange/gateio.py | 8 ++++++++ tests/exchange/test_gateio.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 tests/exchange/test_gateio.py diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index e6ee01c8a..018248a99 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -2,6 +2,7 @@ import logging from typing import Dict +from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange @@ -23,3 +24,10 @@ class Gateio(Exchange): } _headers = {'X-Gate-Channel-Id': 'freqtrade'} + + def validate_ordertypes(self, order_types: Dict) -> None: + super().validate_ordertypes(order_types) + + if any(v == 'market' for k, v in order_types.items()): + raise OperationalException( + f'Exchange {self.name} does not support market orders.') diff --git a/tests/exchange/test_gateio.py b/tests/exchange/test_gateio.py new file mode 100644 index 000000000..6f7862909 --- /dev/null +++ b/tests/exchange/test_gateio.py @@ -0,0 +1,28 @@ +import pytest + +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import Gateio +from freqtrade.resolvers.exchange_resolver import ExchangeResolver + + +def test_validate_order_types_gateio(default_conf, mocker): + default_conf['exchange']['name'] = 'gateio' + mocker.patch('freqtrade.exchange.Exchange._init_ccxt') + mocker.patch('freqtrade.exchange.Exchange._load_markets', return_value={}) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs') + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + mocker.patch('freqtrade.exchange.Exchange.name', 'Bittrex') + exch = ExchangeResolver.load_exchange('gateio', default_conf, True) + assert isinstance(exch, Gateio) + + default_conf['order_types'] = { + 'buy': 'market', + 'sell': 'limit', + 'stoploss': 'market', + 'stoploss_on_exchange': False + } + + with pytest.raises(OperationalException, + match=r'Exchange .* does not support market orders.'): + ExchangeResolver.load_exchange('gateio', default_conf, True) From 0d9beaa3f36a23641d17ee3a7dd2e77933b39846 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 03:01:04 +0000 Subject: [PATCH 194/239] Bump filelock from 3.0.12 to 3.3.0 Bumps [filelock](https://github.com/tox-dev/py-filelock) from 3.0.12 to 3.3.0. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Commits](https://github.com/tox-dev/py-filelock/compare/v3.0.12...3.3.0) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 9feec80f1..b4067d1db 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -5,7 +5,7 @@ scipy==1.7.1 scikit-learn==0.24.2 scikit-optimize==0.8.1 -filelock==3.0.12 +filelock==3.3.0 joblib==1.0.1 psutil==5.8.0 progressbar2==3.53.3 From d220c55d405460de1eed9f5ae9e0dc214e6d8996 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 03:01:11 +0000 Subject: [PATCH 195/239] Bump pymdown-extensions from 8.2 to 9.0 Bumps [pymdown-extensions](https://github.com/facelessuser/pymdown-extensions) from 8.2 to 9.0. - [Release notes](https://github.com/facelessuser/pymdown-extensions/releases) - [Commits](https://github.com/facelessuser/pymdown-extensions/compare/8.2...9.0) --- updated-dependencies: - dependency-name: pymdown-extensions dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 9b7c12a43..67d2c9da8 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.2 mkdocs-material==7.3.0 mdx_truly_sane_lists==1.2 -pymdown-extensions==8.2 +pymdown-extensions==9.0 From ff45d52d497629340d9424728da87060fc369c64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 03:01:14 +0000 Subject: [PATCH 196/239] Bump types-filelock from 0.1.5 to 3.2.0 Bumps [types-filelock](https://github.com/python/typeshed) from 0.1.5 to 3.2.0. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-filelock dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 2f03255a0..3d45247c1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,6 +22,6 @@ nbconvert==6.2.0 # mypy types types-cachetools==4.2.0 -types-filelock==0.1.5 +types-filelock==3.2.0 types-requests==2.25.9 types-tabulate==0.8.2 From 35c4a0a188c0a928600be63a177739d4fb1e42cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 03:01:17 +0000 Subject: [PATCH 197/239] Bump jsonschema from 3.2.0 to 4.0.1 Bumps [jsonschema](https://github.com/Julian/jsonschema) from 3.2.0 to 4.0.1. - [Release notes](https://github.com/Julian/jsonschema/releases) - [Changelog](https://github.com/Julian/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/Julian/jsonschema/compare/v3.2.0...v4.0.1) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index feeb4d942..370766f8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ cachetools==4.2.2 requests==2.26.0 urllib3==1.26.7 wrapt==1.12.1 -jsonschema==3.2.0 +jsonschema==4.0.1 TA-Lib==0.4.21 technical==1.3.0 tabulate==0.8.9 From 0071d002b68c244716ba3010deb110bfbeaad2a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 03:01:22 +0000 Subject: [PATCH 198/239] Bump ccxt from 1.57.3 to 1.57.38 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.57.3 to 1.57.38. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.57.3...1.57.38) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index feeb4d942..5006d356f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.21.2 pandas==1.3.3 pandas-ta==0.3.14b -ccxt==1.57.3 +ccxt==1.57.38 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.8 aiohttp==3.7.4.post0 From 2b41066ab7ccbe032586f231449f82704d37301f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 03:01:29 +0000 Subject: [PATCH 199/239] Bump pytest-cov from 2.12.1 to 3.0.0 Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 2.12.1 to 3.0.0. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v2.12.1...v3.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 2f03255a0..109413e6e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ flake8-tidy-imports==4.4.1 mypy==0.910 pytest==6.2.5 pytest-asyncio==0.15.1 -pytest-cov==2.12.1 +pytest-cov==3.0.0 pytest-mock==3.6.1 pytest-random-order==1.0.4 isort==5.9.3 From 949f4fbbbfdbae9987d212edae7b9a6a3c1db00c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 04:36:11 +0000 Subject: [PATCH 200/239] Bump types-cachetools from 4.2.0 to 4.2.2 Bumps [types-cachetools](https://github.com/python/typeshed) from 4.2.0 to 4.2.2. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-cachetools dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 3d45247c1..858a46389 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -21,7 +21,7 @@ time-machine==2.4.0 nbconvert==6.2.0 # mypy types -types-cachetools==4.2.0 +types-cachetools==4.2.2 types-filelock==3.2.0 types-requests==2.25.9 types-tabulate==0.8.2 From f41fd4e88d7ca79294c40942a5fd181f574469d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 04:40:24 +0000 Subject: [PATCH 201/239] Bump mkdocs-material from 7.3.0 to 7.3.1 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 7.3.0 to 7.3.1. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/docs/changelog.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/7.3.0...7.3.1) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 67d2c9da8..bbbb240ba 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.2 -mkdocs-material==7.3.0 +mkdocs-material==7.3.1 mdx_truly_sane_lists==1.2 pymdown-extensions==9.0 From f15922a16858247c6225d24dd69ae89ceb3e6034 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 4 Oct 2021 19:11:35 +0200 Subject: [PATCH 202/239] Fix custom_stoploss in strategy template closes #5658 --- .../templates/subtemplates/strategy_methods_advanced.j2 | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index 2df23f365..fb467ecaa 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -32,8 +32,7 @@ def custom_stake_amount(self, pair: str, current_time: 'datetime', current_rate: use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: """ Custom stoploss logic, returning the new distance relative to current_rate (as ratio). e.g. returning -0.05 would create a stoploss 5% below current_rate. @@ -44,14 +43,13 @@ def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', When not implemented by a strategy, returns the initial stoploss value Only called when use_custom_stoploss is set to True. - :param pair: Pair that's about to be sold. + :param pair: Pair that's currently analyzed :param trade: trade object. :param current_time: datetime object, containing the current datetime :param current_rate: Rate, calculated based on pricing settings in ask_strategy. :param current_profit: Current profit (as ratio), calculated based on current_rate. - :param dataframe: Analyzed dataframe for this pair. Can contain future data in backtesting. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. - :return float: New stoploss value, relative to the currentrate + :return float: New stoploss value, relative to the current_rate """ return self.stoploss From 7f4baab420ce98c97deeedb69170c3129828c9b2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 4 Oct 2021 20:14:14 +0200 Subject: [PATCH 203/239] Remove explicit rateLimits, improve docs --- config_examples/config_binance.example.json | 4 +- config_examples/config_ftx.example.json | 7 +-- config_examples/config_full.example.json | 8 +-- config_examples/config_kraken.example.json | 4 +- docs/configuration.md | 39 --------------- docs/exchanges.md | 50 +++++++++++++++++++ docs/faq.md | 10 ++-- .../subtemplates/exchange_binance.j2 | 7 +-- .../subtemplates/exchange_generic.j2 | 6 +-- 9 files changed, 65 insertions(+), 70 deletions(-) diff --git a/config_examples/config_binance.example.json b/config_examples/config_binance.example.json index 938bc9342..d59ff96cb 100644 --- a/config_examples/config_binance.example.json +++ b/config_examples/config_binance.example.json @@ -28,10 +28,8 @@ "name": "binance", "key": "your_exchange_key", "secret": "your_exchange_secret", - "ccxt_config": {"enableRateLimit": true}, + "ccxt_config": {}, "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 200 }, "pair_whitelist": [ "ALGO/BTC", diff --git a/config_examples/config_ftx.example.json b/config_examples/config_ftx.example.json index 48651f04c..4d9633cc0 100644 --- a/config_examples/config_ftx.example.json +++ b/config_examples/config_ftx.example.json @@ -28,11 +28,8 @@ "name": "ftx", "key": "your_exchange_key", "secret": "your_exchange_secret", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 50 - }, + "ccxt_config": {}, + "ccxt_async_config": {}, "pair_whitelist": [ "BTC/USD", "ETH/USD", diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index c415d70b0..83b8a27d0 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -84,12 +84,8 @@ "key": "your_exchange_key", "secret": "your_exchange_secret", "password": "", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 500, - "aiohttp_trust_env": false - }, + "ccxt_config": {}, + "ccxt_async_config": {}, "pair_whitelist": [ "ALGO/BTC", "ATOM/BTC", diff --git a/config_examples/config_kraken.example.json b/config_examples/config_kraken.example.json index bf3548568..32def895c 100644 --- a/config_examples/config_kraken.example.json +++ b/config_examples/config_kraken.example.json @@ -28,10 +28,8 @@ "name": "kraken", "key": "your_exchange_key", "secret": "your_exchange_key", - "ccxt_config": {"enableRateLimit": true}, + "ccxt_config": {}, "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 1000 }, "pair_whitelist": [ "ADA/EUR", diff --git a/docs/configuration.md b/docs/configuration.md index 6ccea4c73..bc8a40dcb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -447,45 +447,6 @@ The possible values are: `gtc` (default), `fok` or `ioc`. This is ongoing work. For now, it is supported only for binance and kucoin. Please don't change the default value unless you know what you are doing and have researched the impact of using different values for your particular exchange. -### Exchange configuration - -Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports over 100 cryptocurrency -exchange markets and trading APIs. The complete up-to-date list can be found in the -[CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). - However, the bot was tested by the development team with only Bittrex, Binance and Kraken, - so these are the only officially supported exchanges: - -- [Bittrex](https://bittrex.com/): "bittrex" -- [Binance](https://www.binance.com/): "binance" -- [Kraken](https://kraken.com/): "kraken" - -Feel free to test other exchanges and submit your PR to improve the bot. - -Some exchanges require special configuration, which can be found on the [Exchange-specific Notes](exchanges.md) documentation page. - -#### Sample exchange configuration - -A exchange configuration for "binance" would look as follows: - -```json -"exchange": { - "name": "binance", - "key": "your_exchange_key", - "secret": "your_exchange_secret", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 200 - }, -``` - -This configuration enables binance, as well as rate-limiting to avoid bans from the exchange. -`"rateLimit": 200` defines a wait-event of 0.2s between each call. This can also be completely disabled by setting `"enableRateLimit"` to false. - -!!! Note - Optimal settings for rate-limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings. - We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. - ### What values can be used for fiat_display_currency? The `fiat_display_currency` configuration parameter sets the base currency to use for the diff --git a/docs/exchanges.md b/docs/exchanges.md index c0fbdc694..badaa484a 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -2,6 +2,56 @@ This page combines common gotchas and informations which are exchange-specific and most likely don't apply to other exchanges. +## Exchange configuration + +Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports over 100 cryptocurrency +exchange markets and trading APIs. The complete up-to-date list can be found in the +[CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). +However, the bot was tested by the development team with only a few exchanges. +A current list of these can be found in the "Home" section of this documentation. + +Feel free to test other exchanges and submit your feedback or PR to improve the bot or confirm exchanges that work flawlessly.. + +Some exchanges require special configuration, which can be found below. + +### Sample exchange configuration + +A exchange configuration for "binance" would look as follows: + +```json +"exchange": { + "name": "binance", + "key": "your_exchange_key", + "secret": "your_exchange_secret", + "ccxt_config": {}, + "ccxt_async_config": {}, + // ... +``` + +### Setting rate limits + +Usually, rate limits set by CCXT are reliable and work well. +In case of problems related to rate-limits (usually DDOS Exceptions in your logs), it's easy to change rateLimit settings to other values. + +```json +"exchange": { + "name": "kraken", + "key": "your_exchange_key", + "secret": "your_exchange_secret", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 3100 + }, +``` + +This configuration enables kraken, as well as rate-limiting to avoid bans from the exchange. +`"rateLimit": 3100` defines a wait-event of 0.2s between each call. This can also be completely disabled by setting `"enableRateLimit"` to false. + +!!! Note + Optimal settings for rate-limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings. + We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. + ## Binance Binance supports [time_in_force](configuration.md#understand-order_time_in_force). diff --git a/docs/faq.md b/docs/faq.md index 285625491..75c40a681 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -82,11 +82,11 @@ Currently known to happen for US Bittrex users. Read [the Bittrex section about restricted markets](exchanges.md#restricted-markets) for more information. -### I'm getting the "Exchange Bittrex does not support market orders." message and cannot run my strategy +### I'm getting the "Exchange XXX does not support market orders." message and cannot run my strategy -As the message says, Bittrex does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Your strategy was probably written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex). +As the message says, your exchange does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Your strategy was probably written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex and Gate.io). -To fix it for Bittrex, redefine order types in the strategy to use "limit" instead of "market": +To fix this, redefine order types in the strategy to use "limit" instead of "market": ``` order_types = { @@ -136,6 +136,8 @@ On Windows, the `--logfile` option is also supported by Freqtrade and you can us > type \path\to\mylogfile.log | findstr "something" ``` +## Hyperopt module + ### Why does freqtrade not have GPU support? First of all, most indicator libraries don't have GPU support - as such, there would be little benefit for indicator calculations. @@ -152,8 +154,6 @@ The benefit of using GPU would therefore be pretty slim - and will not justify t There is however nothing preventing you from using GPU-enabled indicators within your strategy if you think you must have this - you will however probably be disappointed by the slim gain that will give you (compared to the complexity). -## Hyperopt module - ### How many epochs do I need to get a good Hyperopt result? Per default Hyperopt called without the `-e`/`--epochs` command line option will only diff --git a/freqtrade/templates/subtemplates/exchange_binance.j2 b/freqtrade/templates/subtemplates/exchange_binance.j2 index de58b6f72..dc2272119 100644 --- a/freqtrade/templates/subtemplates/exchange_binance.j2 +++ b/freqtrade/templates/subtemplates/exchange_binance.j2 @@ -2,11 +2,8 @@ "name": "{{ exchange_name | lower }}", "key": "{{ exchange_key }}", "secret": "{{ exchange_secret }}", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 200 - }, + "ccxt_config": {}, + "ccxt_async_config": {}, "pair_whitelist": [ ], "pair_blacklist": [ diff --git a/freqtrade/templates/subtemplates/exchange_generic.j2 b/freqtrade/templates/subtemplates/exchange_generic.j2 index ade9c2f28..08b11f365 100644 --- a/freqtrade/templates/subtemplates/exchange_generic.j2 +++ b/freqtrade/templates/subtemplates/exchange_generic.j2 @@ -2,10 +2,8 @@ "name": "{{ exchange_name | lower }}", "key": "{{ exchange_key }}", "secret": "{{ exchange_secret }}", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true - }, + "ccxt_config": {}, + "ccxt_async_config": {}, "pair_whitelist": [ ], From 92f8f231afe79eafa5079d494b56db71e7f30baf Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 4 Oct 2021 20:22:41 +0200 Subject: [PATCH 204/239] Remove ratelimit from kucoin template --- freqtrade/templates/subtemplates/exchange_kucoin.j2 | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/freqtrade/templates/subtemplates/exchange_kucoin.j2 b/freqtrade/templates/subtemplates/exchange_kucoin.j2 index 9882c51c7..b797dda41 100644 --- a/freqtrade/templates/subtemplates/exchange_kucoin.j2 +++ b/freqtrade/templates/subtemplates/exchange_kucoin.j2 @@ -3,14 +3,8 @@ "key": "{{ exchange_key }}", "secret": "{{ exchange_secret }}", "password": "{{ exchange_key_password }}", - "ccxt_config": { - "enableRateLimit": true, - "rateLimit": 200 - }, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 200 - }, + "ccxt_config": {}, + "ccxt_async_config": {}, "pair_whitelist": [ ], "pair_blacklist": [ From 0db5c07314a02beb7dde0baf76a06fff458fd5c6 Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 5 Oct 2021 00:10:39 +0100 Subject: [PATCH 205/239] Fix issues with sysinfo rpc/API code, add SysInfo api_schema --- freqtrade/rpc/api_server/api_schemas.py | 4 ++++ freqtrade/rpc/api_server/api_v1.py | 8 ++++---- freqtrade/rpc/rpc.py | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 46187f571..b03400900 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -347,3 +347,7 @@ class BacktestResponse(BaseModel): trade_count: Optional[float] # TODO: Properly type backtestresult... backtest_result: Optional[Dict[str, Any]] + +class SysInfo(BaseModel): + cpu_pct: float + ram_pct: float diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 733fa7383..d52e8c10d 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -18,7 +18,7 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac OpenTradeSchema, PairHistory, PerformanceEntry, Ping, PlotConfig, Profit, ResultMsg, ShowConfig, Stats, StatusMsg, StrategyListResponse, - StrategyResponse, Version, WhitelistResponse) + StrategyResponse, SysInfo, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException @@ -260,6 +260,6 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option } return result -@router.get('/sysinfo', tags=['info']) -def sysinfo(rpc: RPC = Depends(get_rpc)): - return rpc._rpc_sysinfo() +@router.get('/sysinfo', response_model=SysInfo, tags=['info']) +def sysinfo(): + return RPC._rpc_sysinfo() diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 9b0d4b0f7..699e3b384 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -871,5 +871,6 @@ class RPC: self._freqtrade.strategy.plot_config['subplots'] = {} return self._freqtrade.strategy.plot_config - def _rpc_sysinfo(self) -> Dict[str, Any]: + @staticmethod + def _rpc_sysinfo() -> Dict[str, Any]: return {"cpu_pct": psutil.cpu_percent(interval=1, percpu=True), "ram_pct": psutil.virtual_memory().percent} From 949d616082c49be11211a396de676077ad741c8d Mon Sep 17 00:00:00 2001 From: jonny07 Date: Tue, 5 Oct 2021 21:33:15 +0200 Subject: [PATCH 206/239] Update docker_quickstart.md Got help in the discord chat to get the UI running, I think most people will need this... --- docs/docker_quickstart.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 27a9091b1..0fe69933a 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -70,6 +70,40 @@ docker-compose up -d !!! Warning "Default configuration" While the configuration generated will be mostly functional, you will still need to verify that all options correspond to what you want (like Pricing, pairlist, ...) before starting the bot. +#### Acessing the UI + +Uncommend the 2 lines below and add your IP adress in the following format (like 192.168.2.67:8080:8080) to the ft_userdata/docker-compose.yml: +'''bash + ports: + - "yourIPadress:8080:8080" +''' +Your ft_userdata/user_data/config.json should look like: +'''bash +api_server": { + "enabled": true, + "listen_ip_address": "0.0.0.0", + "listen_port": 8080, + "verbosity": "error", + "enable_openapi": false, + "jwt_secret_key": "****", + "CORS_origins": [], + "username": "****", + "password": "****" + }, +''' +instead of "****" you will have your data in. +Then rebuild your docker file: +Linux: +'''bash +sudo docker-compose down && sudo docker-compose pull && sudo docker-compose build && sudo docker-compose up -d +''' +Windows: +'''bash +docker-compose down && docker-compose pull && docker-compose build && docker-compose up -d +''' + +You can now access the UI by typing yourIPadress:8080 in your browser. + #### Monitoring the bot You can check for running instances with `docker-compose ps`. From c0d01dbc26a1552562ca339a92111f6193e7a02c Mon Sep 17 00:00:00 2001 From: sid Date: Wed, 6 Oct 2021 13:24:27 +0530 Subject: [PATCH 207/239] add max_drawdown loss --- .../optimize/hyperopt_loss_max_drawdown.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 freqtrade/optimize/hyperopt_loss_max_drawdown.py diff --git a/freqtrade/optimize/hyperopt_loss_max_drawdown.py b/freqtrade/optimize/hyperopt_loss_max_drawdown.py new file mode 100644 index 000000000..e6f73e04a --- /dev/null +++ b/freqtrade/optimize/hyperopt_loss_max_drawdown.py @@ -0,0 +1,42 @@ +""" +MaxDrawDownHyperOptLoss + +This module defines the alternative HyperOptLoss class which can be used for +Hyperoptimization. +""" +from datetime import datetime +from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.optimize.hyperopt import IHyperOptLoss + +from pandas import DataFrame + + +class MaxDrawDownHyperOptLoss(IHyperOptLoss): + + """ + Defines the loss function for hyperopt. + + This implementation optimizes for max draw down and profit + Less max drawdown more profit -> Lower return value + """ + + @staticmethod + def hyperopt_loss_function(results: DataFrame, trade_count: int, + min_date: datetime, max_date: datetime, + *args, **kwargs) -> float: + + """ + Objective function. + + Uses profit ratio weighted max_drawdown when drawdown is available. + Otherwise directly optimizes profit ratio. + """ + total_profit = results['profit_ratio'].sum() + try: + max_drawdown = calculate_max_drawdown(results) + except ValueError: + # No losing trade, therefore no drawdown. + return -total_profit + max_drawdown_rev = 1 / max_drawdown[0] + ret = max_drawdown_rev * total_profit + return -ret \ No newline at end of file From 6ba46b38bdd434ddd65b065c6393beb0b29aa492 Mon Sep 17 00:00:00 2001 From: sid Date: Wed, 6 Oct 2021 13:46:05 +0530 Subject: [PATCH 208/239] fix formatting --- freqtrade/optimize/hyperopt_loss_max_drawdown.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss_max_drawdown.py b/freqtrade/optimize/hyperopt_loss_max_drawdown.py index e6f73e04a..4fa32c00e 100644 --- a/freqtrade/optimize/hyperopt_loss_max_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss_max_drawdown.py @@ -5,11 +5,12 @@ This module defines the alternative HyperOptLoss class which can be used for Hyperoptimization. """ from datetime import datetime -from freqtrade.data.btanalysis import calculate_max_drawdown -from freqtrade.optimize.hyperopt import IHyperOptLoss from pandas import DataFrame +from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.optimize.hyperopt import IHyperOptLoss + class MaxDrawDownHyperOptLoss(IHyperOptLoss): @@ -31,7 +32,7 @@ class MaxDrawDownHyperOptLoss(IHyperOptLoss): Uses profit ratio weighted max_drawdown when drawdown is available. Otherwise directly optimizes profit ratio. """ - total_profit = results['profit_ratio'].sum() + total_profit = results['profit_ratio'].sum() try: max_drawdown = calculate_max_drawdown(results) except ValueError: @@ -39,4 +40,4 @@ class MaxDrawDownHyperOptLoss(IHyperOptLoss): return -total_profit max_drawdown_rev = 1 / max_drawdown[0] ret = max_drawdown_rev * total_profit - return -ret \ No newline at end of file + return -ret From 57ef25789e3db1dd59de1e76096bf730fefdf0d7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Oct 2021 19:36:28 +0200 Subject: [PATCH 209/239] Fix style errors --- freqtrade/rpc/api_server/api_schemas.py | 1 + freqtrade/rpc/api_server/api_v1.py | 4 +++- freqtrade/rpc/rpc.py | 8 ++++++-- scripts/rest_client.py | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index b03400900..bde6af35b 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -348,6 +348,7 @@ class BacktestResponse(BaseModel): # TODO: Properly type backtestresult... backtest_result: Optional[Dict[str, Any]] + class SysInfo(BaseModel): cpu_pct: float ram_pct: float diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index d52e8c10d..06230a7db 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -18,7 +18,8 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac OpenTradeSchema, PairHistory, PerformanceEntry, Ping, PlotConfig, Profit, ResultMsg, ShowConfig, Stats, StatusMsg, StrategyListResponse, - StrategyResponse, SysInfo, Version, WhitelistResponse) + StrategyResponse, SysInfo, Version, + WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException @@ -260,6 +261,7 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option } return result + @router.get('/sysinfo', response_model=SysInfo, tags=['info']) def sysinfo(): return RPC._rpc_sysinfo() diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 699e3b384..d0858350c 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -1,13 +1,14 @@ """ This module contains class to define a RPC communications """ -import logging, psutil +import logging from abc import abstractmethod from datetime import date, datetime, timedelta, timezone from math import isnan from typing import Any, Dict, List, Optional, Tuple, Union import arrow +import psutil from numpy import NAN, inf, int64, mean from pandas import DataFrame @@ -873,4 +874,7 @@ class RPC: @staticmethod def _rpc_sysinfo() -> Dict[str, Any]: - return {"cpu_pct": psutil.cpu_percent(interval=1, percpu=True), "ram_pct": psutil.virtual_memory().percent} + return { + "cpu_pct": psutil.cpu_percent(interval=1, percpu=True), + "ram_pct": psutil.virtual_memory().percent + } diff --git a/scripts/rest_client.py b/scripts/rest_client.py index 52de3c534..ac3b6defe 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -341,6 +341,7 @@ class FtRestClient(): """ return self._get("sysinfo") + def add_arguments(): parser = argparse.ArgumentParser() parser.add_argument("command", From 992cef56e653844c4b8613c683db81057959f569 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Oct 2021 19:36:51 +0200 Subject: [PATCH 210/239] Add test for sysinfo endpoint --- tests/rpc/test_rpc_apiserver.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 7c98b2df7..117b1fa49 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1271,6 +1271,16 @@ def test_list_available_pairs(botclient): assert len(rc.json()['pair_interval']) == 1 +def test_sysinfo(botclient): + ftbot, client = botclient + + rc = client_get(client, f"{BASE_URI}/sysinfo") + assert_response(rc) + result = rc.json() + assert 'cpu_pct' in result + assert 'ram_pct' in result + + def test_api_backtesting(botclient, mocker, fee, caplog): ftbot, client = botclient mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) From 65d4df938df14881729a63bf2c58ace1aca57d22 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Oct 2021 20:09:08 +0200 Subject: [PATCH 211/239] Improve docker port api --- docker-compose.yml | 6 +++--- docs/docker_quickstart.md | 43 ++++++++++++++++++++------------------- docs/rest-api.md | 2 +- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 80e194ab2..445fbaea0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,10 +15,10 @@ services: volumes: - "./user_data:/freqtrade/user_data" # Expose api on port 8080 (localhost only) - # Please read the https://www.freqtrade.io/en/latest/rest-api/ documentation + # Please read the https://www.freqtrade.io/en/stable/rest-api/ documentation # before enabling this. - # ports: - # - "127.0.0.1:8080:8080" + ports: + - "127.0.0.1:8080:8080" # Default command used when running `docker compose up` command: > trade diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 27a9091b1..d5bec54c9 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -148,27 +148,9 @@ You'll then also need to modify the `docker-compose.yml` file and uncomment the dockerfile: "./Dockerfile." ``` -You can then run `docker-compose build` to build the docker image, and run it using the commands described above. +You can then run `docker-compose build --pull` to build the docker image, and run it using the commands described above. -### Troubleshooting - -#### Docker on Windows - -* Error: `"Timestamp for this request is outside of the recvWindow."` - * The market api requests require a synchronized clock but the time in the docker container shifts a bit over time into the past. - To fix this issue temporarily you need to run `wsl --shutdown` and restart docker again (a popup on windows 10 will ask you to do so). - A permanent solution is either to host the docker container on a linux host or restart the wsl from time to time with the scheduler. - ``` - taskkill /IM "Docker Desktop.exe" /F - wsl --shutdown - start "" "C:\Program Files\Docker\Docker\Docker Desktop.exe" - ``` - -!!! Warning - Due to the above, we do not recommend the usage of docker on windows for production setups, but only for experimentation, datadownload and backtesting. - Best use a linux-VPS for running freqtrade reliably. - -## Plotting with docker-compose +### Plotting with docker-compose Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your docker-compose.yml file. You can then use these commands as follows: @@ -179,7 +161,7 @@ docker-compose run --rm freqtrade plot-dataframe --strategy AwesomeStrategy -p B The output will be stored in the `user_data/plot` directory, and can be opened with any modern browser. -## Data analysis using docker compose +### Data analysis using docker compose Freqtrade provides a docker-compose file which starts up a jupyter lab server. You can run this server using the following command: @@ -196,3 +178,22 @@ Since part of this image is built on your machine, it is recommended to rebuild ``` bash docker-compose -f docker/docker-compose-jupyter.yml build --no-cache ``` + +## Troubleshooting + +### Docker on Windows + +* Error: `"Timestamp for this request is outside of the recvWindow."` + * The market api requests require a synchronized clock but the time in the docker container shifts a bit over time into the past. + To fix this issue temporarily you need to run `wsl --shutdown` and restart docker again (a popup on windows 10 will ask you to do so). + A permanent solution is either to host the docker container on a linux host or restart the wsl from time to time with the scheduler. + + ``` bash + taskkill /IM "Docker Desktop.exe" /F + wsl --shutdown + start "" "C:\Program Files\Docker\Docker\Docker Desktop.exe" + ``` + +!!! Warning + Due to the above, we do not recommend the usage of docker on windows for production setups, but only for experimentation, datadownload and backtesting. + Best use a linux-VPS for running freqtrade reliably. diff --git a/docs/rest-api.md b/docs/rest-api.md index b9b2b29be..b4992e047 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -78,7 +78,7 @@ If you run your bot using docker, you'll need to have the bot listen to incoming }, ``` -Uncomment the following from your docker-compose file: +Make sure that the following 2 lines are available in your docker-compose file: ```yml ports: From 526bdaa2dc04cd84dbea35c72482697cc865080e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Oct 2021 20:14:59 +0200 Subject: [PATCH 212/239] Recommend using 0.0.0.0 as listen address for docker --- freqtrade/commands/build_config_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index faa8a98f4..34ae35aff 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -163,7 +163,8 @@ def ask_user_config() -> Dict[str, Any]: { "type": "text", "name": "api_server_listen_addr", - "message": "Insert Api server Listen Address (best left untouched default!)", + "message": ("Insert Api server Listen Address (0.0.0.0 for docker, " + "otherwise best left untouched)"), "default": "127.0.0.1", "when": lambda x: x['api_server'] }, From 46c320513aa0b825b370034feba9d6e9f29af312 Mon Sep 17 00:00:00 2001 From: sid Date: Thu, 7 Oct 2021 08:07:07 +0530 Subject: [PATCH 213/239] use profit_abs --- freqtrade/optimize/hyperopt_loss_max_drawdown.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss_max_drawdown.py b/freqtrade/optimize/hyperopt_loss_max_drawdown.py index 4fa32c00e..6777fb2e8 100644 --- a/freqtrade/optimize/hyperopt_loss_max_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss_max_drawdown.py @@ -32,9 +32,9 @@ class MaxDrawDownHyperOptLoss(IHyperOptLoss): Uses profit ratio weighted max_drawdown when drawdown is available. Otherwise directly optimizes profit ratio. """ - total_profit = results['profit_ratio'].sum() + total_profit = results['profit_abs'].sum() try: - max_drawdown = calculate_max_drawdown(results) + max_drawdown = calculate_max_drawdown(results, value_col='profit_abs') except ValueError: # No losing trade, therefore no drawdown. return -total_profit From 29863ad2bf7d364e0bc17dc5c3506e24f20b31b4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Oct 2021 06:51:29 +0200 Subject: [PATCH 214/239] Fix error when ask_last_balance is not set closes #5181 --- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_exchange.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index e9d0316d2..7cc436430 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1058,7 +1058,7 @@ class Exchange: ticker_rate = ticker[conf_strategy['price_side']] if ticker['last'] and ticker_rate: if side == 'buy' and ticker_rate > ticker['last']: - balance = conf_strategy['ask_last_balance'] + balance = conf_strategy.get('ask_last_balance', 0.0) ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) elif side == 'sell' and ticker_rate < ticker['last']: balance = conf_strategy.get('bid_last_balance', 0.0) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 691cf3c03..8cb494edb 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1832,6 +1832,7 @@ def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name): ('ask', 20, 19, 10, 0.3, 17), # Between ask and last ('ask', 5, 6, 10, 1.0, 5), # last bigger than ask ('ask', 5, 6, 10, 0.5, 5), # last bigger than ask + ('ask', 20, 19, 10, None, 20), # ask_last_balance missing ('ask', 10, 20, None, 0.5, 10), # last not available - uses ask ('ask', 4, 5, None, 0.5, 4), # last not available - uses ask ('ask', 4, 5, None, 1, 4), # last not available - uses ask @@ -1842,6 +1843,7 @@ def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name): ('bid', 21, 20, 10, 0.7, 13), # Between bid and last ('bid', 21, 20, 10, 0.3, 17), # Between bid and last ('bid', 6, 5, 10, 1.0, 5), # last bigger than bid + ('bid', 21, 20, 10, None, 20), # ask_last_balance missing ('bid', 6, 5, 10, 0.5, 5), # last bigger than bid ('bid', 21, 20, None, 0.5, 20), # last not available - uses bid ('bid', 6, 5, None, 0.5, 5), # last not available - uses bid @@ -1851,7 +1853,10 @@ def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name): def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, last, last_ab, expected) -> None: caplog.set_level(logging.DEBUG) - default_conf['bid_strategy']['ask_last_balance'] = last_ab + if last_ab is None: + del default_conf['bid_strategy']['ask_last_balance'] + else: + default_conf['bid_strategy']['ask_last_balance'] = last_ab default_conf['bid_strategy']['price_side'] = side exchange = get_patched_exchange(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', @@ -1876,6 +1881,7 @@ def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, ('bid', 12.0, 11.2, 10.5, 1.0, 11.2), # Last smaller than bid - uses bid ('bid', 12.0, 11.2, 10.5, 0.5, 11.2), # Last smaller than bid - uses bid ('bid', 0.003, 0.002, 0.005, 0.0, 0.002), + ('bid', 0.003, 0.002, 0.005, None, 0.002), ('ask', 12.0, 11.0, 12.5, 0.0, 12.0), # full ask side ('ask', 12.0, 11.0, 12.5, 1.0, 12.5), # full last side ('ask', 12.0, 11.0, 12.5, 0.5, 12.25), # between bid and lat @@ -1886,6 +1892,7 @@ def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, ('ask', 10.11, 11.2, 11.0, 0.0, 10.11), ('ask', 0.001, 0.002, 11.0, 0.0, 0.001), ('ask', 0.006, 1.0, 11.0, 0.0, 0.006), + ('ask', 0.006, 1.0, 11.0, None, 0.006), ]) def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, last, last_ab, expected) -> None: From 45b7a0c8377a5493496abafa70e48b0872d0b4b5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Oct 2021 07:12:45 +0200 Subject: [PATCH 215/239] Add Test and docs for MaxDrawDownHyperOptLoss --- docs/hyperopt.md | 18 ++++++++++-------- freqtrade/constants.py | 3 ++- tests/optimize/test_hyperoptloss.py | 5 +++-- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 09d43939a..45e0d444d 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -60,7 +60,7 @@ optional arguments: Specify what timerange of data to use. --data-format-ohlcv {json,jsongz,hdf5} Storage format for downloaded candle (OHLCV) data. - (default: `None`). + (default: `json`). --max-open-trades INT Override the value of the `max_open_trades` configuration setting. @@ -114,7 +114,8 @@ optional arguments: Hyperopt-loss-functions are: ShortTradeDurHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily, - SortinoHyperOptLoss, SortinoHyperOptLossDaily + SortinoHyperOptLoss, SortinoHyperOptLossDaily, + MaxDrawDownHyperOptLoss --disable-param-export Disable automatic hyperopt parameter export. @@ -512,12 +513,13 @@ This class should be in its own file within the `user_data/hyperopts/` directory Currently, the following loss functions are builtin: -* `ShortTradeDurHyperOptLoss` (default legacy Freqtrade hyperoptimization loss function) - Mostly for short trade duration and avoiding losses. -* `OnlyProfitHyperOptLoss` (which takes only amount of profit into consideration) -* `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on trade returns relative to standard deviation) -* `SharpeHyperOptLossDaily` (optimizes Sharpe Ratio calculated on **daily** trade returns relative to standard deviation) -* `SortinoHyperOptLoss` (optimizes Sortino Ratio calculated on trade returns relative to **downside** standard deviation) -* `SortinoHyperOptLossDaily` (optimizes Sortino Ratio calculated on **daily** trade returns relative to **downside** standard deviation) +* `ShortTradeDurHyperOptLoss` - (default legacy Freqtrade hyperoptimization loss function) - Mostly for short trade duration and avoiding losses. +* `OnlyProfitHyperOptLoss` - takes only amount of profit into consideration. +* `SharpeHyperOptLoss` - optimizes Sharpe Ratio calculated on trade returns relative to standard deviation. +* `SharpeHyperOptLossDaily` - optimizes Sharpe Ratio calculated on **daily** trade returns relative to standard deviation. +* `SortinoHyperOptLoss` - optimizes Sortino Ratio calculated on trade returns relative to **downside** standard deviation. +* `SortinoHyperOptLossDaily` - optimizes Sortino Ratio calculated on **daily** trade returns relative to **downside** standard deviation. +* `MaxDrawDownHyperOptLoss` - Optimizes Maximum drawdown. Creation of a custom loss function is covered in the [Advanced Hyperopt](advanced-hyperopt.md) part of the documentation. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index fca319a0f..c6b8f0e62 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -24,7 +24,8 @@ ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily', - 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] + 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily', + 'MaxDrawDownHyperOptLoss'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'OffsetFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', diff --git a/tests/optimize/test_hyperoptloss.py b/tests/optimize/test_hyperoptloss.py index 923e3fc32..a39190934 100644 --- a/tests/optimize/test_hyperoptloss.py +++ b/tests/optimize/test_hyperoptloss.py @@ -84,13 +84,14 @@ def test_loss_calculation_has_limited_profit(hyperopt_conf, hyperopt_results) -> "SortinoHyperOptLossDaily", "SharpeHyperOptLoss", "SharpeHyperOptLossDaily", + "MaxDrawDownHyperOptLoss", ]) def test_loss_functions_better_profits(default_conf, hyperopt_results, lossfunction) -> None: results_over = hyperopt_results.copy() - results_over['profit_abs'] = hyperopt_results['profit_abs'] * 2 + results_over['profit_abs'] = hyperopt_results['profit_abs'] * 2 + 0.2 results_over['profit_ratio'] = hyperopt_results['profit_ratio'] * 2 results_under = hyperopt_results.copy() - results_under['profit_abs'] = hyperopt_results['profit_abs'] / 2 + results_under['profit_abs'] = hyperopt_results['profit_abs'] / 2 - 0.2 results_under['profit_ratio'] = hyperopt_results['profit_ratio'] / 2 default_conf.update({'hyperopt_loss': lossfunction}) From a1be6124f221d6e3bd294c376a76848ff3e6d197 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Oct 2021 07:15:09 +0200 Subject: [PATCH 216/239] Don't set bid_last_balance if None in tests part of #5681 --- tests/exchange/test_exchange.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 8cb494edb..e3369182d 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1899,7 +1899,8 @@ def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, caplog.set_level(logging.DEBUG) default_conf['ask_strategy']['price_side'] = side - default_conf['ask_strategy']['bid_last_balance'] = last_ab + if last_ab is not None: + default_conf['ask_strategy']['bid_last_balance'] = last_ab mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value={'ask': ask, 'bid': bid, 'last': last}) pair = "ETH/BTC" From f07eeddda0e912e44467bff42b6c08f125902ae1 Mon Sep 17 00:00:00 2001 From: Robert Davey Date: Thu, 7 Oct 2021 12:04:42 +0100 Subject: [PATCH 217/239] Update api_schemas.py Fix api schema for cpu_pct float List. --- freqtrade/rpc/api_server/api_schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index bde6af35b..e9985c3c6 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -350,5 +350,5 @@ class BacktestResponse(BaseModel): class SysInfo(BaseModel): - cpu_pct: float + cpu_pct: List[float] ram_pct: float From 1327c21d0103eee5e040e03e598662c993bcfac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Thu, 7 Oct 2021 19:12:09 +0530 Subject: [PATCH 218/239] Update README.md --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 01effd7bc..79763983d 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,7 @@ Please find the complete documentation on our [website](https://www.freqtrade.io Freqtrade provides a Linux/macOS script to install all dependencies and help you to configure the bot. ```bash -git clone -b develop https://github.com/freqtrade/freqtrade.git -cd freqtrade -./setup.sh --install +pip install freqtrade ``` For any other type of installation please refer to [Installation doc](https://www.freqtrade.io/en/latest/installation/). From 482f4418c68c2b99635a7af3cc65a14c1022e031 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Oct 2021 14:36:52 +0200 Subject: [PATCH 219/239] Clarify "required candle" message --- freqtrade/exchange/exchange.py | 62 ++++++++++++++-------------------- 1 file changed, 25 insertions(+), 37 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 7cc436430..b6cfb8d8b 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -26,9 +26,9 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun InvalidOrderException, OperationalException, PricingError, RetryableOrderError, TemporaryError) from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, - EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, - remove_credentials, retrier, retrier_async) -from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2 + EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier, + retrier_async) +from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -54,16 +54,12 @@ class Exchange: # Parameters to add directly to buy/sell calls (like agreeing to trading agreement) _params: Dict = {} - # Additional headers - added to the ccxt object - _headers: Dict = {} - # Dict to specify which options each exchange implements # This defines defaults, which can be selectively overridden by subclasses using _ft_has # or by specifying them in the configuration. _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], - "time_in_force_parameter": "timeInForce", "ohlcv_params": {}, "ohlcv_candle_limit": 500, "ohlcv_partial_candle": True, @@ -104,7 +100,6 @@ class Exchange: # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} - remove_credentials(config) if config['dry_run']: logger.info('Instance is running with dry_run enabled') @@ -174,7 +169,7 @@ class Exchange: asyncio.get_event_loop().run_until_complete(self._api_async.close()) def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt, - ccxt_kwargs: Dict = {}) -> ccxt.Exchange: + ccxt_kwargs: dict = None) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid ccxt instance. @@ -193,10 +188,6 @@ class Exchange: } if ccxt_kwargs: logger.info('Applying additional ccxt config: %s', ccxt_kwargs) - if self._headers: - # Inject static headers after the above output to not confuse users. - ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs) - if ccxt_kwargs: ex_config.update(ccxt_kwargs) try: @@ -480,7 +471,7 @@ class Exchange: if startup_candles + 5 > candle_limit: raise OperationalException( f"This strategy requires {startup_candles} candles to start. " - f"{self.name} only provides {candle_limit} for {timeframe}.") + f"{self.name} only provides {candle_limit - 5} for {timeframe}.") def exchange_has(self, endpoint: str) -> bool: """ @@ -523,7 +514,7 @@ class Exchange: precision = self.markets[pair]['precision']['price'] missing = price % precision if missing != 0: - price = round(price - missing + precision, 10) + price = price - missing + precision else: symbol_prec = self.markets[pair]['precision']['price'] big_price = price * pow(10, symbol_prec) @@ -725,8 +716,7 @@ class Exchange: params = self._params.copy() if time_in_force != 'gtc' and ordertype != 'market': - param = self._ft_has.get('time_in_force_parameter', '') - params.update({param: time_in_force}) + params.update({'timeInForce': time_in_force}) try: # Set the precision for amount and price(rate) as accepted by the exchange @@ -1058,7 +1048,7 @@ class Exchange: ticker_rate = ticker[conf_strategy['price_side']] if ticker['last'] and ticker_rate: if side == 'buy' and ticker_rate > ticker['last']: - balance = conf_strategy.get('ask_last_balance', 0.0) + balance = conf_strategy['ask_last_balance'] ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) elif side == 'sell' and ticker_rate < ticker['last']: balance = conf_strategy.get('bid_last_balance', 0.0) @@ -1195,7 +1185,7 @@ class Exchange: # Historic data def get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool = False) -> List: + since_ms: int) -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1207,7 +1197,7 @@ class Exchange: """ return asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms, is_new_pair=is_new_pair)) + since_ms=since_ms)) def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, since_ms: int) -> DataFrame: @@ -1222,12 +1212,11 @@ class Exchange: return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) - async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool - ) -> List: + async def _async_get_historic_ohlcv(self, pair: str, + timeframe: str, + since_ms: int) -> List: """ Download historic ohlcv - :param is_new_pair: used by binance subclass to allow "fast" new pair downloading """ one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) @@ -1240,22 +1229,21 @@ class Exchange: pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] - data: List = [] - # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling - for input_coro in chunks(input_coroutines, 100): + results = await asyncio.gather(*input_coroutines, return_exceptions=True) - results = await asyncio.gather(*input_coro, return_exceptions=True) - for res in results: - if isinstance(res, Exception): - logger.warning("Async code raised an exception: %s", res.__class__.__name__) - continue - # Deconstruct tuple if it's not an exception - p, _, new_data = res - if p == pair: - data.extend(new_data) + # Combine gathered results + data: List = [] + for res in results: + if isinstance(res, Exception): + logger.warning("Async code raised an exception: %s", res.__class__.__name__) + continue + # Deconstruct tuple if it's not an exception + p, _, new_data = res + if p == pair: + data.extend(new_data) # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) - logger.info(f"Downloaded data for {pair} with length {len(data)}.") + logger.info("Downloaded data for %s with length %s.", pair, len(data)) return data def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, From 11ec1d9b062ae7a063d8b6549cdfd5e468047d63 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Oct 2021 20:22:07 +0200 Subject: [PATCH 220/239] Revert previous commit --- freqtrade/exchange/exchange.py | 60 ++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b6cfb8d8b..4143b79a5 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -26,9 +26,9 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun InvalidOrderException, OperationalException, PricingError, RetryableOrderError, TemporaryError) from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, - EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier, - retrier_async) -from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 + EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, + remove_credentials, retrier, retrier_async) +from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -54,12 +54,16 @@ class Exchange: # Parameters to add directly to buy/sell calls (like agreeing to trading agreement) _params: Dict = {} + # Additional headers - added to the ccxt object + _headers: Dict = {} + # Dict to specify which options each exchange implements # This defines defaults, which can be selectively overridden by subclasses using _ft_has # or by specifying them in the configuration. _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], + "time_in_force_parameter": "timeInForce", "ohlcv_params": {}, "ohlcv_candle_limit": 500, "ohlcv_partial_candle": True, @@ -100,6 +104,7 @@ class Exchange: # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} + remove_credentials(config) if config['dry_run']: logger.info('Instance is running with dry_run enabled') @@ -169,7 +174,7 @@ class Exchange: asyncio.get_event_loop().run_until_complete(self._api_async.close()) def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt, - ccxt_kwargs: dict = None) -> ccxt.Exchange: + ccxt_kwargs: Dict = {}) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid ccxt instance. @@ -188,6 +193,10 @@ class Exchange: } if ccxt_kwargs: logger.info('Applying additional ccxt config: %s', ccxt_kwargs) + if self._headers: + # Inject static headers after the above output to not confuse users. + ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs) + if ccxt_kwargs: ex_config.update(ccxt_kwargs) try: @@ -514,7 +523,7 @@ class Exchange: precision = self.markets[pair]['precision']['price'] missing = price % precision if missing != 0: - price = price - missing + precision + price = round(price - missing + precision, 10) else: symbol_prec = self.markets[pair]['precision']['price'] big_price = price * pow(10, symbol_prec) @@ -716,7 +725,8 @@ class Exchange: params = self._params.copy() if time_in_force != 'gtc' and ordertype != 'market': - params.update({'timeInForce': time_in_force}) + param = self._ft_has.get('time_in_force_parameter', '') + params.update({param: time_in_force}) try: # Set the precision for amount and price(rate) as accepted by the exchange @@ -1048,7 +1058,7 @@ class Exchange: ticker_rate = ticker[conf_strategy['price_side']] if ticker['last'] and ticker_rate: if side == 'buy' and ticker_rate > ticker['last']: - balance = conf_strategy['ask_last_balance'] + balance = conf_strategy.get('ask_last_balance', 0.0) ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) elif side == 'sell' and ticker_rate < ticker['last']: balance = conf_strategy.get('bid_last_balance', 0.0) @@ -1185,7 +1195,7 @@ class Exchange: # Historic data def get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int) -> List: + since_ms: int, is_new_pair: bool = False) -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1197,7 +1207,7 @@ class Exchange: """ return asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms)) + since_ms=since_ms, is_new_pair=is_new_pair)) def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, since_ms: int) -> DataFrame: @@ -1212,11 +1222,12 @@ class Exchange: return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) - async def _async_get_historic_ohlcv(self, pair: str, - timeframe: str, - since_ms: int) -> List: + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool + ) -> List: """ Download historic ohlcv + :param is_new_pair: used by binance subclass to allow "fast" new pair downloading """ one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) @@ -1229,21 +1240,22 @@ class Exchange: pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] - results = await asyncio.gather(*input_coroutines, return_exceptions=True) - - # Combine gathered results data: List = [] - for res in results: - if isinstance(res, Exception): - logger.warning("Async code raised an exception: %s", res.__class__.__name__) - continue - # Deconstruct tuple if it's not an exception - p, _, new_data = res - if p == pair: - data.extend(new_data) + # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling + for input_coro in chunks(input_coroutines, 100): + + results = await asyncio.gather(*input_coro, return_exceptions=True) + for res in results: + if isinstance(res, Exception): + logger.warning("Async code raised an exception: %s", res.__class__.__name__) + continue + # Deconstruct tuple if it's not an exception + p, _, new_data = res + if p == pair: + data.extend(new_data) # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) - logger.info("Downloaded data for %s with length %s.", pair, len(data)) + logger.info(f"Downloaded data for {pair} with length {len(data)}.") return data def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, From 30bc96cf3f802a882e4e5e35e0c0df8db876acfc Mon Sep 17 00:00:00 2001 From: sid Date: Sat, 9 Oct 2021 06:36:23 +0530 Subject: [PATCH 221/239] simplify expression --- freqtrade/optimize/hyperopt_loss_max_drawdown.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss_max_drawdown.py b/freqtrade/optimize/hyperopt_loss_max_drawdown.py index 6777fb2e8..ce955d928 100644 --- a/freqtrade/optimize/hyperopt_loss_max_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss_max_drawdown.py @@ -38,6 +38,4 @@ class MaxDrawDownHyperOptLoss(IHyperOptLoss): except ValueError: # No losing trade, therefore no drawdown. return -total_profit - max_drawdown_rev = 1 / max_drawdown[0] - ret = max_drawdown_rev * total_profit - return -ret + return -total_profit / max_drawdown[0] From 7b1c888665b214aee599958bf0a5b9656d13531b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Oct 2021 08:39:32 +0200 Subject: [PATCH 222/239] Add FAQ entry for incomplete candles closes #5687 --- docs/faq.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 75c40a681..d9777ddf1 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -54,9 +54,11 @@ you can't say much from few trades. Yes. You can edit your config and use the `/reload_config` command to reload the configuration. The bot will stop, reload the configuration and strategy and will restart with the new configuration and strategy. -### I want to improve the bot with a new strategy +### I want to use incomplete candles -That's great. We have a nice backtesting and hyperoptimization setup. See the tutorial [here|Testing-new-strategies-with-Hyperopt](bot-usage.md#hyperopt-commands). +Freqtrade will not provide incomplete candles to strategies. Using incomplete candles will lead to repainting and consequently to strategies with "ghost" buys, which are impossible to both backtest, and verify after they happened. + +You can use "current" market data by using the [dataprovider](strategy-customization.md#orderbookpair-maximum)'s orderbook or ticker methods - which however cannot be used during backtesting. ### Is there a setting to only SELL the coins being held and not perform anymore BUYS? From 2c68342140979f7ade3e271d6841b9be81590b51 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Oct 2021 10:37:33 +0200 Subject: [PATCH 223/239] Move pypi installation to documentation --- README.md | 8 +++++--- docs/installation.md | 7 +++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 79763983d..0a4d6424e 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Please find the complete documentation on our [website](https://www.freqtrade.io - [x] **Dry-run**: Run the bot without paying money. - [x] **Backtesting**: Run a simulation of your buy/sell strategy. - [x] **Strategy Optimization by machine learning**: Use machine learning to optimize your buy/sell strategy parameters with real exchange data. -- [x] **Edge position sizing** Calculate your win rate, risk reward ratio, the best stoploss and adjust your position size before taking a position for each specific market. [Learn more](https://www.freqtrade.io/en/latest/edge/). +- [x] **Edge position sizing** Calculate your win rate, risk reward ratio, the best stoploss and adjust your position size before taking a position for each specific market. [Learn more](https://www.freqtrade.io/en/stable/edge/). - [x] **Whitelist crypto-currencies**: Select which crypto-currency you want to trade or use dynamic whitelists. - [x] **Blacklist crypto-currencies**: Select which crypto-currency you want to avoid. - [x] **Manageable via Telegram**: Manage the bot with Telegram. @@ -66,10 +66,12 @@ Please find the complete documentation on our [website](https://www.freqtrade.io Freqtrade provides a Linux/macOS script to install all dependencies and help you to configure the bot. ```bash -pip install freqtrade +git clone -b develop https://github.com/freqtrade/freqtrade.git +cd freqtrade +./setup.sh --install ``` -For any other type of installation please refer to [Installation doc](https://www.freqtrade.io/en/latest/installation/). +For any other type of installation please refer to [Installation doc](https://www.freqtrade.io/en/stable/installation/). ## Basic Usage diff --git a/docs/installation.md b/docs/installation.md index 5e4a19d88..d468786d3 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -113,6 +113,13 @@ git checkout develop You may later switch between branches at any time with the `git checkout stable`/`git checkout develop` commands. +??? Note "Install from pypi" + An alternative way to install Freqtrade is from [pypi](https://pypi.org/project/freqtrade/). The downside is that this method requires ta-lib to be correctly installed beforehand, and is therefore currently not the recommended way to install Freqtrade. + + ``` bash + pip install freqtrade + ``` + ------ ## Script Installation From 1a3b41ed9718338c6458de3f87527591337a03cb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Oct 2021 15:35:39 +0200 Subject: [PATCH 224/239] Rephrase and simplify UI access section in docker quickstart --- docs/docker_quickstart.md | 40 +++++++++------------------------------ 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index cf525b926..95df37811 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -70,39 +70,17 @@ docker-compose up -d !!! Warning "Default configuration" While the configuration generated will be mostly functional, you will still need to verify that all options correspond to what you want (like Pricing, pairlist, ...) before starting the bot. -#### Acessing the UI +#### Accessing the UI -Uncommend the 2 lines below and add your IP adress in the following format (like 192.168.2.67:8080:8080) to the ft_userdata/docker-compose.yml: -'''bash - ports: - - "yourIPadress:8080:8080" -''' -Your ft_userdata/user_data/config.json should look like: -'''bash -api_server": { - "enabled": true, - "listen_ip_address": "0.0.0.0", - "listen_port": 8080, - "verbosity": "error", - "enable_openapi": false, - "jwt_secret_key": "****", - "CORS_origins": [], - "username": "****", - "password": "****" - }, -''' -instead of "****" you will have your data in. -Then rebuild your docker file: -Linux: -'''bash -sudo docker-compose down && sudo docker-compose pull && sudo docker-compose build && sudo docker-compose up -d -''' -Windows: -'''bash -docker-compose down && docker-compose pull && docker-compose build && docker-compose up -d -''' +If you've selected to enable FreqUI in the `new-config` step, you will have freqUI available at port `localhost:8080`. -You can now access the UI by typing yourIPadress:8080 in your browser. +You can now access the UI by typing localhost:8080 in your browser. + +??? Note "UI Access on a remote servers" + If you're running on a VPS, you should consider using either a ssh tunnel, or setup a VPN (openVPN, wireguard) to connect to your bot. + This will ensure that freqUI is not directly exposed to the internet, which is not recommended for security reasons (freqUI does not support https out of the box). + Setup of these tools is not part of this tutorial, however many good tutorials can be found on the internet. + Please also read the [API configuration with docker](rest-api.md#configuration-with-docker) section to learn more about this configuration. #### Monitoring the bot From 57095d7167aa481bf184f19f39acfeded8149a7e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 03:01:22 +0000 Subject: [PATCH 225/239] Bump wrapt from 1.12.1 to 1.13.1 Bumps [wrapt](https://github.com/GrahamDumpleton/wrapt) from 1.12.1 to 1.13.1. - [Release notes](https://github.com/GrahamDumpleton/wrapt/releases) - [Changelog](https://github.com/GrahamDumpleton/wrapt/blob/develop/docs/changes.rst) - [Commits](https://github.com/GrahamDumpleton/wrapt/compare/1.12.1...1.13.1) --- updated-dependencies: - dependency-name: wrapt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7fe06b9d2..4eb2bf66b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ arrow==1.1.1 cachetools==4.2.2 requests==2.26.0 urllib3==1.26.7 -wrapt==1.12.1 +wrapt==1.13.1 jsonschema==4.0.1 TA-Lib==0.4.21 technical==1.3.0 From 7323ffa25a408a64a94ce577caeece6530bdfa1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 03:01:31 +0000 Subject: [PATCH 226/239] Bump blosc from 1.10.4 to 1.10.6 Bumps [blosc](https://github.com/blosc/python-blosc) from 1.10.4 to 1.10.6. - [Release notes](https://github.com/blosc/python-blosc/releases) - [Changelog](https://github.com/Blosc/python-blosc/blob/master/RELEASE_NOTES.rst) - [Commits](https://github.com/blosc/python-blosc/compare/v1.10.4...v1.10.6) --- updated-dependencies: - dependency-name: blosc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7fe06b9d2..3a49977c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ tabulate==0.8.9 pycoingecko==2.2.0 jinja2==3.0.1 tables==3.6.1 -blosc==1.10.4 +blosc==1.10.6 # find first, C search in arrays py_find_1st==1.1.5 From 5fb0401dca6b9ae19c678b0a7577129969901ff9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 03:01:39 +0000 Subject: [PATCH 227/239] Bump cryptography from 3.4.8 to 35.0.0 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.4.8 to 35.0.0. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.4.8...35.0.0) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7fe06b9d2..89728f782 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ pandas-ta==0.3.14b ccxt==1.57.38 # Pin cryptography for now due to rust build errors with piwheels -cryptography==3.4.8 +cryptography==35.0.0 aiohttp==3.7.4.post0 SQLAlchemy==1.4.25 python-telegram-bot==13.7 From 3fdc62d29c3f81f4ebdd826e0cd5ad21cde6d1a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 05:05:14 +0000 Subject: [PATCH 228/239] Bump flake8 from 3.9.2 to 4.0.0 Bumps [flake8](https://github.com/pycqa/flake8) from 3.9.2 to 4.0.0. - [Release notes](https://github.com/pycqa/flake8/releases) - [Commits](https://github.com/pycqa/flake8/compare/3.9.2...4.0.0) --- updated-dependencies: - dependency-name: flake8 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index a3ed37bea..3f2d6035d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ -r requirements-hyperopt.txt coveralls==3.2.0 -flake8==3.9.2 +flake8==4.0.0 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.4.1 mypy==0.910 From e467491dbecc7f49444c60471485daacf546dfa7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 05:05:23 +0000 Subject: [PATCH 229/239] Bump jsonschema from 4.0.1 to 4.1.0 Bumps [jsonschema](https://github.com/Julian/jsonschema) from 4.0.1 to 4.1.0. - [Release notes](https://github.com/Julian/jsonschema/releases) - [Changelog](https://github.com/Julian/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/Julian/jsonschema/compare/v4.0.1...v4.1.0) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4eb2bf66b..641264d53 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ cachetools==4.2.2 requests==2.26.0 urllib3==1.26.7 wrapt==1.13.1 -jsonschema==4.0.1 +jsonschema==4.1.0 TA-Lib==0.4.21 technical==1.3.0 tabulate==0.8.9 From afc086f33c85162107a27dda013544d37ced3a51 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 05:05:30 +0000 Subject: [PATCH 230/239] Bump arrow from 1.1.1 to 1.2.0 Bumps [arrow](https://github.com/arrow-py/arrow) from 1.1.1 to 1.2.0. - [Release notes](https://github.com/arrow-py/arrow/releases) - [Changelog](https://github.com/arrow-py/arrow/blob/master/CHANGELOG.rst) - [Commits](https://github.com/arrow-py/arrow/commits) --- updated-dependencies: - dependency-name: arrow dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4eb2bf66b..90ef02ae5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ cryptography==3.4.8 aiohttp==3.7.4.post0 SQLAlchemy==1.4.25 python-telegram-bot==13.7 -arrow==1.1.1 +arrow==1.2.0 cachetools==4.2.2 requests==2.26.0 urllib3==1.26.7 From 32174f8f906304f61e7f21afa3669eab4458dc5d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 05:14:08 +0000 Subject: [PATCH 231/239] Bump pyjwt from 2.1.0 to 2.2.0 Bumps [pyjwt](https://github.com/jpadilla/pyjwt) from 2.1.0 to 2.2.0. - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jpadilla/pyjwt/compare/2.1.0...2.2.0) --- updated-dependencies: - dependency-name: pyjwt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4eb2bf66b..133aee144 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ tabulate==0.8.9 pycoingecko==2.2.0 jinja2==3.0.1 tables==3.6.1 -blosc==1.10.4 +blosc==1.10.6 # find first, C search in arrays py_find_1st==1.1.5 @@ -34,7 +34,7 @@ sdnotify==0.3.2 # API Server fastapi==0.68.1 uvicorn==0.15.0 -pyjwt==2.1.0 +pyjwt==2.2.0 aiofiles==0.7.0 psutil==5.8.0 From 4921a4caecd0cfbffa7df3894f4649876d4dd28a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 05:14:19 +0000 Subject: [PATCH 232/239] Bump jinja2 from 3.0.1 to 3.0.2 Bumps [jinja2](https://github.com/pallets/jinja) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.0.1...3.0.2) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1d9aaf4b5..e2de4f629 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ TA-Lib==0.4.21 technical==1.3.0 tabulate==0.8.9 pycoingecko==2.2.0 -jinja2==3.0.1 +jinja2==3.0.2 tables==3.6.1 blosc==1.10.6 From 90ea3d444060359320ec534044785699617e0cb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 05:18:24 +0000 Subject: [PATCH 233/239] Bump mkdocs-material from 7.3.1 to 7.3.2 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 7.3.1 to 7.3.2. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/7.3.1...7.3.2) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index bbbb240ba..9a733d8f7 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.2 -mkdocs-material==7.3.1 +mkdocs-material==7.3.2 mdx_truly_sane_lists==1.2 pymdown-extensions==9.0 From 29371b2f28591a611e039bad4da718ae726dfb23 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 06:15:03 +0000 Subject: [PATCH 234/239] Bump joblib from 1.0.1 to 1.1.0 Bumps [joblib](https://github.com/joblib/joblib) from 1.0.1 to 1.1.0. - [Release notes](https://github.com/joblib/joblib/releases) - [Changelog](https://github.com/joblib/joblib/blob/master/CHANGES.rst) - [Commits](https://github.com/joblib/joblib/compare/1.0.1...1.1.0) --- updated-dependencies: - dependency-name: joblib dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index b4067d1db..96690bcbb 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -6,6 +6,6 @@ scipy==1.7.1 scikit-learn==0.24.2 scikit-optimize==0.8.1 filelock==3.3.0 -joblib==1.0.1 +joblib==1.1.0 psutil==5.8.0 progressbar2==3.53.3 From 802599bdc99cdd82e390d51cdb000c87d03e96a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 06:15:32 +0000 Subject: [PATCH 235/239] Bump ccxt from 1.57.38 to 1.57.94 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.57.38 to 1.57.94. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.57.38...1.57.94) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 788801987..ff9ea9423 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.21.2 pandas==1.3.3 pandas-ta==0.3.14b -ccxt==1.57.38 +ccxt==1.57.94 # Pin cryptography for now due to rust build errors with piwheels cryptography==35.0.0 aiohttp==3.7.4.post0 @@ -34,7 +34,7 @@ sdnotify==0.3.2 # API Server fastapi==0.68.1 uvicorn==0.15.0 -pyjwt==2.1.0 +pyjwt==2.2.0 aiofiles==0.7.0 psutil==5.8.0 From fa00b52c4742cbe7b2ec4c583ee5a5828ca00ff7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 07:41:16 +0000 Subject: [PATCH 236/239] Bump scikit-learn from 0.24.2 to 1.0 Bumps [scikit-learn](https://github.com/scikit-learn/scikit-learn) from 0.24.2 to 1.0. - [Release notes](https://github.com/scikit-learn/scikit-learn/releases) - [Commits](https://github.com/scikit-learn/scikit-learn/compare/0.24.2...1.0) --- updated-dependencies: - dependency-name: scikit-learn dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 96690bcbb..edd078e9e 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -3,7 +3,7 @@ # Required for hyperopt scipy==1.7.1 -scikit-learn==0.24.2 +scikit-learn==1.0 scikit-optimize==0.8.1 filelock==3.3.0 joblib==1.1.0 From 396bc9b2e3d33993052a3b2038cad9ee19d3736d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 11 Oct 2021 20:00:53 +0200 Subject: [PATCH 237/239] Version bump flake8-tidy-imports to 4.5.0 --- requirements-dev.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 3f2d6035d..74ebee479 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,8 +5,7 @@ coveralls==3.2.0 flake8==4.0.0 -flake8-type-annotations==0.1.0 -flake8-tidy-imports==4.4.1 +flake8-tidy-imports==4.5.0 mypy==0.910 pytest==6.2.5 pytest-asyncio==0.15.1 From ce9debe9fd89bab9171f35d3632f88f27c0d080a Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 12 Oct 2021 06:44:07 +0200 Subject: [PATCH 238/239] Add version argument to freqUI installer --- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 6 ++++++ freqtrade/commands/deploy_commands.py | 16 ++++++++++++---- tests/commands/test_commands.py | 27 ++++++++++++++++++++++----- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 9643705a5..a02faa736 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -73,7 +73,7 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source", "timeframe", "plot_auto_open"] -ARGS_INSTALL_UI = ["erase_ui_only"] +ARGS_INSTALL_UI = ["erase_ui_only", 'ui_version'] ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index d350a9426..30a9b0137 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -414,6 +414,12 @@ AVAILABLE_CLI_OPTIONS = { action='store_true', default=False, ), + "ui_version": Arg( + '--ui-version', + help=('Specify a specific version of FreqUI to install. ' + 'Not specifying this installs the latest version.'), + type=str, + ), # Templating options "template": Arg( '--template', diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 4f9e5bbad..92c9adf66 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -128,7 +128,7 @@ def download_and_install_ui(dest_folder: Path, dl_url: str, version: str): f.write(version) -def get_ui_download_url() -> Tuple[str, str]: +def get_ui_download_url(version: Optional[str] = None) -> Tuple[str, str]: base_url = 'https://api.github.com/repos/freqtrade/frequi/' # Get base UI Repo path @@ -136,8 +136,16 @@ def get_ui_download_url() -> Tuple[str, str]: resp.raise_for_status() r = resp.json() - latest_version = r[0]['name'] - assets = r[0].get('assets', []) + if version: + tmp = [x for x in r if x['name'] == version] + if tmp: + latest_version = tmp[0]['name'] + assets = tmp[0].get('assets', []) + else: + raise ValueError("UI-Version not found.") + else: + latest_version = r[0]['name'] + assets = r[0].get('assets', []) dl_url = '' if assets and len(assets) > 0: dl_url = assets[0]['browser_download_url'] @@ -156,7 +164,7 @@ def start_install_ui(args: Dict[str, Any]) -> None: dest_folder = Path(__file__).parents[1] / 'rpc/api_server/ui/installed/' # First make sure the assets are removed. - dl_url, latest_version = get_ui_download_url() + dl_url, latest_version = get_ui_download_url(args.get('ui_version')) curr_version = read_ui_version(dest_folder) if curr_version == latest_version and not args.get('erase_ui_only'): diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 8889617ba..6a0e741d9 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -605,16 +605,33 @@ def test_get_ui_download_url(mocker): def test_get_ui_download_url_direct(mocker): response = MagicMock() response.json = MagicMock( - side_effect=[[{ - 'assets_url': 'http://whatever.json', - 'name': '0.0.1', - 'assets': [{'browser_download_url': 'http://download11.zip'}]}]]) + return_value=[ + { + 'assets_url': 'http://whatever.json', + 'name': '0.0.2', + 'assets': [{'browser_download_url': 'http://download22.zip'}] + }, + { + 'assets_url': 'http://whatever.json', + 'name': '0.0.1', + 'assets': [{'browser_download_url': 'http://download1.zip'}] + }, + ]) get_mock = mocker.patch("freqtrade.commands.deploy_commands.requests.get", return_value=response) x, last_version = get_ui_download_url() assert get_mock.call_count == 1 + assert last_version == '0.0.2' + assert x == 'http://download22.zip' + get_mock.reset_mock() + response.json.reset_mock() + + x, last_version = get_ui_download_url('0.0.1') assert last_version == '0.0.1' - assert x == 'http://download11.zip' + assert x == 'http://download1.zip' + + with pytest.raises(ValueError, match="UI-Version not found."): + x, last_version = get_ui_download_url('0.0.3') def test_download_data_keyboardInterrupt(mocker, caplog, markets): From 8798ae567744ceefed92577acbd66fb974df6b15 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 12 Oct 2021 19:06:23 +0200 Subject: [PATCH 239/239] Version bump also scikit-optimize --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index edd078e9e..e97e78638 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -4,7 +4,7 @@ # Required for hyperopt scipy==1.7.1 scikit-learn==1.0 -scikit-optimize==0.8.1 +scikit-optimize==0.9.0 filelock==3.3.0 joblib==1.1.0 psutil==5.8.0