diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index d0f3f0df6..c415d70b0 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -149,7 +149,9 @@ }, "sell_fill": "on", "buy_cancel": "on", - "sell_cancel": "on" + "sell_cancel": "on", + "protection_trigger": "off", + "protection_trigger_global": "on" }, "reload": true, "balance_dust_level": 0.01 diff --git a/docs/data-download.md b/docs/data-download.md index 0ca86b0d3..5f605c404 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -204,6 +204,61 @@ It'll also remove original jsongz data files (`--erase` parameter). freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase ``` +### Sub-command trades to ohlcv + +When you need to use `--dl-trades` (kraken only) to download data, conversion of trades data to ohlcv data is the last step. +This command will allow you to repeat this last step for additional timeframes without re-downloading the data. + +``` +usage: freqtrade trades-to-ohlcv [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [-p PAIRS [PAIRS ...]] + [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]] + [--exchange EXCHANGE] + [--data-format-ohlcv {json,jsongz,hdf5}] + [--data-format-trades {json,jsongz,hdf5}] + +optional arguments: + -h, --help show this help message and exit + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Limit command to these pairs. Pairs are space- + separated. + -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...] + Specify which tickers to download. Space-separated + list. Default: `1m 5m`. + --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no + config is provided. + --data-format-ohlcv {json,jsongz,hdf5} + Storage format for downloaded candle (OHLCV) data. + (default: `json`). + --data-format-trades {json,jsongz,hdf5} + Storage format for downloaded trades data. (default: + `jsongz`). + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + +``` + +#### Example trade-to-ohlcv conversion + +``` bash +freqtrade trades-to-ohlcv --exchange kraken -t 5m 1h 1d --pairs BTC/EUR ETH/EUR +``` + ### Sub-command list-data You can get a list of downloaded data using the `list-data` sub-command. diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 1fa229225..2f350d207 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -149,6 +149,24 @@ You'll then also need to modify the `docker-compose.yml` file and uncomment the You can then run `docker-compose build` to build the docker image, and run it using the commands described above. +### Troubleshooting + +#### Docker on Windows + +* Error: `"Timestamp for this request is outside of the recvWindow."` + * The market api requests require a synchronized clock but the time in the docker container shifts a bit over time into the past. + To fix this issue temporarily you need to run `wsl --shutdown` and restart docker again (a popup on windows 10 will ask you to do so). + A permanent solution is either to host the docker container on a linux host or restart the wsl from time to time with the scheduler. + ``` + taskkill /IM "Docker Desktop.exe" /F + wsl --shutdown + start "" "C:\Program Files\Docker\Docker\Docker Desktop.exe" + ``` + +!!! Warning + Due to the above, we do not recommend the usage of docker on windows for production setups, but only for experimentation, datadownload and backtesting. + Best use a linux-VPS for running freqtrade reliably. + ## Plotting with docker-compose Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your docker-compose.yml file. diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 9927740c2..9b7c12a43 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.2 -mkdocs-material==7.2.6 +mkdocs-material==7.3.0 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index dc1e2831a..96ec04515 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -731,3 +731,33 @@ The variable 'content', will contain the strategy file in a BASE64 encoded form. ``` Please ensure that 'NameOfStrategy' is identical to the strategy name! + +## Performance warning + +When executing a strategy, one can sometimes be greeted by the following in the logs + +> PerformanceWarning: DataFrame is highly fragmented. + +This is a warning from [`pandas`](https://github.com/pandas-dev/pandas) and as the warning continues to say: +use `pd.concat(axis=1)`. +This can have slight performance implications, which are usually only visible during hyperopt (when optimizing an indicator). + +For example: + +```python +for val in self.buy_ema_short.range: + dataframe[f'ema_short_{val}'] = ta.EMA(dataframe, timeperiod=val) +``` + +should be rewritten to + +```python +frames = [dataframe] +for val in self.buy_ema_short.range: + frames.append({ + f'ema_short_{val}': ta.EMA(dataframe, timeperiod=val) + }) + +# Append columns to existing dataframe +merged_frame = pd.concat(frames, axis=1) +``` diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 725252b30..0bfc0a2f6 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -122,6 +122,16 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame Look into the [user_data/strategies/sample_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_strategy.py). Then uncomment indicators you need. +#### Indicator libraries + +Out of the box, freqtrade installs the following technical libraries: + +* [ta-lib](http://mrjbq7.github.io/ta-lib/) +* [pandas-ta](https://twopirllc.github.io/pandas-ta/) +* [technical](https://github.com/freqtrade/technical/) + +Additional technical libraries can be installed as necessary, or custom indicators may be written / invented by the strategy author. + ### Strategy startup period Most indicators have an instable startup period, in which they are either not available, or the calculation is incorrect. This can lead to inconsistencies, since Freqtrade does not know how long this instable period should be. @@ -942,6 +952,8 @@ Printing more than a few rows is also possible (simply use `print(dataframe)` i ## Common mistakes when developing strategies +### Peeking into the future while backtesting + Backtesting analyzes the whole time-range at once for performance reasons. Because of this, strategy authors need to make sure that strategies do not look-ahead into the future. This is a common pain-point, which can cause huge differences between backtesting and dry/live run methods, since they all use data which is not available during dry/live runs, so these strategies will perform well during backtesting, but will fail / perform badly in real conditions. diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index b020b00db..b9d01a236 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -93,7 +93,9 @@ Example configuration showing the different settings: "buy_cancel": "silent", "sell_cancel": "on", "buy_fill": "off", - "sell_fill": "off" + "sell_fill": "off", + "protection_trigger": "off", + "protection_trigger_global": "on" }, "reload": true, "balance_dust_level": 0.01 @@ -103,6 +105,7 @@ Example configuration showing the different settings: `buy` notifications are sent when the order is placed, while `buy_fill` notifications are sent when the order is filled on the exchange. `sell` notifications are sent when the order is placed, while `sell_fill` notifications are sent when the order is filled on the exchange. `*_fill` notifications are off by default and must be explicitly enabled. +`protection_trigger` notifications are sent when a protection triggers and `protection_trigger_global` notifications trigger when global protections are triggered. `balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown. diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index a6f14cff7..858c99acd 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -8,8 +8,8 @@ Note: Be careful with file-scoped imports in these subfiles. """ from freqtrade.commands.arguments import Arguments from freqtrade.commands.build_config_commands import start_new_config -from freqtrade.commands.data_commands import (start_convert_data, start_download_data, - start_list_data) +from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades, + start_download_data, start_list_data) from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui, start_new_strategy) from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index d424f3ce7..9643705a5 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -58,6 +58,8 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] +ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"] + ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange", @@ -91,7 +93,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-data", "hyperopt-list", "hyperopt-show", - "plot-dataframe", "plot-profit", "show-trades"] + "plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] @@ -169,14 +171,14 @@ class Arguments: self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot') self._build_args(optionlist=['version'], parser=self.parser) - from freqtrade.commands import (start_backtesting, start_convert_data, start_create_userdir, - start_download_data, start_edge, start_hyperopt, - start_hyperopt_list, start_hyperopt_show, start_install_ui, - start_list_data, start_list_exchanges, start_list_markets, - start_list_strategies, start_list_timeframes, - start_new_config, start_new_strategy, start_plot_dataframe, - start_plot_profit, start_show_trades, start_test_pairlist, - start_trading, start_webserver) + from freqtrade.commands import (start_backtesting, start_convert_data, start_convert_trades, + start_create_userdir, start_download_data, start_edge, + start_hyperopt, start_hyperopt_list, start_hyperopt_show, + start_install_ui, start_list_data, start_list_exchanges, + start_list_markets, start_list_strategies, + start_list_timeframes, start_new_config, start_new_strategy, + start_plot_dataframe, start_plot_profit, start_show_trades, + start_test_pairlist, start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -236,6 +238,15 @@ class Arguments: convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False)) self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd) + # Add trades-to-ohlcv subcommand + convert_trade_data_cmd = subparsers.add_parser( + 'trades-to-ohlcv', + help='Convert trade data to OHLCV data.', + parents=[_common_parser], + ) + convert_trade_data_cmd.set_defaults(func=start_convert_trades) + self._build_args(optionlist=ARGS_CONVERT_TRADES, parser=convert_trade_data_cmd) + # Add list-data subcommand list_data_cmd = subparsers.add_parser( 'list-data', diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index e3c7fe464..d350a9426 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -381,12 +381,12 @@ AVAILABLE_CLI_OPTIONS = { ), "dataformat_ohlcv": Arg( '--data-format-ohlcv', - help='Storage format for downloaded candle (OHLCV) data. (default: `%(default)s`).', + help='Storage format for downloaded candle (OHLCV) data. (default: `json`).', choices=constants.AVAILABLE_DATAHANDLERS, ), "dataformat_trades": Arg( '--data-format-trades', - help='Storage format for downloaded trades data. (default: `%(default)s`).', + help='Storage format for downloaded trades data. (default: `jsongz`).', choices=constants.AVAILABLE_DATAHANDLERS, ), "exchange": Arg( diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 141e85f14..ee05e6c69 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -89,6 +89,41 @@ def start_download_data(args: Dict[str, Any]) -> None: f"on exchange {exchange.name}.") +def start_convert_trades(args: Dict[str, Any]) -> None: + + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + timerange = TimeRange() + + # Remove stake-currency to skip checks which are not relevant for datadownload + config['stake_currency'] = '' + + if 'pairs' not in config: + raise OperationalException( + "Downloading data requires a list of pairs. " + "Please check the documentation on how to configure this.") + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + # Manual validations of relevant settings + if not config['exchange'].get('skip_pair_validation', False): + exchange.validate_pairs(config['pairs']) + expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets)) + + logger.info(f"About to Convert pairs: {expanded_pairs}, " + f"intervals: {config['timeframes']} to {config['datadir']}") + + for timeframe in config['timeframes']: + exchange.validate_timeframes(timeframe) + # Convert downloaded trade data to different timeframes + convert_trades_to_ohlcv( + pairs=expanded_pairs, timeframes=config['timeframes'], + datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), + data_format_ohlcv=config['dataformat_ohlcv'], + data_format_trades=config['dataformat_trades'], + ) + + def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: """ Convert data from one format to another diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 9ca43d459..fca319a0f 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -110,7 +110,7 @@ CONF_SCHEMA = { }, 'tradable_balance_ratio': { 'type': 'number', - 'minimum': 0.1, + 'minimum': 0.0, 'maximum': 1, 'default': 0.99 }, @@ -284,6 +284,15 @@ CONF_SCHEMA = { 'enum': TELEGRAM_SETTING_OPTIONS, 'default': 'off' }, + 'protection_trigger': { + 'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS, + 'default': 'off' + }, + 'protection_trigger_global': { + 'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS, + }, } }, 'reload': {'type': 'boolean'}, diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index cdee0f078..b197c159f 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -149,6 +149,8 @@ class DataProvider: Clear pair dataframe cache. """ self.__cached_pairs = {} + self.__cached_pairs_backtesting = {} + self.__slice_index = 0 # Exchange functions diff --git a/freqtrade/enums/rpcmessagetype.py b/freqtrade/enums/rpcmessagetype.py index 9c59f6108..4e3f693e5 100644 --- a/freqtrade/enums/rpcmessagetype.py +++ b/freqtrade/enums/rpcmessagetype.py @@ -11,6 +11,8 @@ class RPCMessageType(Enum): SELL = 'sell' SELL_FILL = 'sell_fill' SELL_CANCEL = 'sell_cancel' + PROTECTION_TRIGGER = 'protection_trigger' + PROTECTION_TRIGGER_GLOBAL = 'protection_trigger_global' def __repr__(self): return self.value diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 32edd8588..26edfea87 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -141,7 +141,7 @@ class FreqtradeBot(LoggingMixin): # Only update open orders on startup # This will update the database after the initial migration - self.update_open_orders() + self.startup_update_open_orders() def process(self) -> None: """ @@ -239,7 +239,7 @@ class FreqtradeBot(LoggingMixin): open_trades = len(Trade.get_open_trades()) return max(0, self.config['max_open_trades'] - open_trades) - def update_open_orders(self): + def startup_update_open_orders(self): """ Updates open orders based on order list kept in the database. Mainly updates the state of orders - but may also close trades @@ -1244,7 +1244,7 @@ class FreqtradeBot(LoggingMixin): 'exchange': trade.exchange.capitalize(), 'pair': trade.pair, 'gain': gain, - 'limit': profit_rate, + 'limit': profit_rate or 0, 'order_type': order_type, 'amount': trade.amount, 'open_rate': trade.open_rate, @@ -1253,7 +1253,7 @@ class FreqtradeBot(LoggingMixin): 'profit_ratio': profit_ratio, 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, - 'close_date': trade.close_date, + 'close_date': trade.close_date or datetime.now(timezone.utc), 'stake_currency': self.config['stake_currency'], 'fiat_currency': self.config.get('fiat_display_currency', None), 'reason': reason, @@ -1319,8 +1319,7 @@ class FreqtradeBot(LoggingMixin): if not trade.is_open: if not stoploss_order and not trade.open_order_id: self._notify_exit(trade, '', True) - self.protections.stop_per_pair(trade.pair) - self.protections.global_stop() + self.handle_protections(trade.pair) self.wallets.update() elif not trade.open_order_id: # Buy fill @@ -1328,6 +1327,19 @@ class FreqtradeBot(LoggingMixin): return False + def handle_protections(self, pair: str) -> None: + prot_trig = self.protections.stop_per_pair(pair) + if prot_trig: + msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } + msg.update(prot_trig.to_json()) + self.rpc.send_msg(msg) + + prot_trig_glb = self.protections.global_stop() + if prot_trig_glb: + msg = {'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, } + msg.update(prot_trig_glb.to_json()) + self.rpc.send_msg(msg) + def apply_fee_conditional(self, trade: Trade, trade_base_currency: str, amount: float, fee_abs: float) -> float: """ diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4890c20aa..aaf875a94 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -87,18 +87,7 @@ class Backtesting: "configuration or as cli argument `--timeframe 5m`") self.timeframe = str(self.config.get('timeframe')) self.timeframe_min = timeframe_to_minutes(self.timeframe) - # Load detail timeframe if specified - self.timeframe_detail = str(self.config.get('timeframe_detail', '')) - if self.timeframe_detail: - self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail) - if self.timeframe_min <= self.timeframe_detail_min: - raise OperationalException( - "Detail timeframe must be smaller than strategy timeframe.") - - else: - self.timeframe_detail_min = 0 - self.detail_data: Dict[str, DataFrame] = {} - + self.init_backtest_detail() self.pairlists = PairListManager(self.exchange, self.config) if 'VolumePairList' in self.pairlists.name_list: raise OperationalException("VolumePairList not allowed for backtesting.") @@ -121,14 +110,6 @@ class Backtesting: else: self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0]) - Trade.use_db = False - Trade.reset_trades() - PairLocks.timeframe = self.config['timeframe'] - PairLocks.use_db = False - PairLocks.reset_locks() - - self.wallets = Wallets(self.config, self.exchange, log=False) - self.timerange = TimeRange.parse_timerange( None if self.config.get('timerange') is None else str(self.config.get('timerange'))) @@ -144,6 +125,7 @@ class Backtesting: self.progress = BTProgress() self.abort = False + self.init_backtest() def __del__(self): self.cleanup() @@ -153,6 +135,28 @@ class Backtesting: PairLocks.use_db = True Trade.use_db = True + def init_backtest_detail(self): + # Load detail timeframe if specified + self.timeframe_detail = str(self.config.get('timeframe_detail', '')) + if self.timeframe_detail: + self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail) + if self.timeframe_min <= self.timeframe_detail_min: + raise OperationalException( + "Detail timeframe must be smaller than strategy timeframe.") + + else: + self.timeframe_detail_min = 0 + self.detail_data: Dict[str, DataFrame] = {} + + def init_backtest(self): + + self.prepare_backtest(False) + + self.wallets = Wallets(self.config, self.exchange, log=False) + + self.progress = BTProgress() + self.abort = False + def _set_strategy(self, strategy: IStrategy): """ Load strategy into backtesting @@ -232,7 +236,8 @@ class Backtesting: Trade.reset_trades() self.rejected_trades = 0 self.dataprovider.clear_cache() - self._load_protections(self.strategy) + if enable_protections: + self._load_protections(self.strategy) def check_abort(self): """ @@ -365,7 +370,7 @@ class Backtesting: trade, sell_row[OPEN_IDX], sell_candle_time, # type: ignore enter=enter, exit_=exit_, low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX] - ) + ) if sell.sell_flag: trade.close_date = sell_candle_time @@ -397,14 +402,14 @@ class Backtesting: detail_data = detail_data.loc[ (detail_data['date'] >= sell_candle_time) & (detail_data['date'] < sell_candle_end) - ] + ].copy() if len(detail_data) == 0: # Fall back to "regular" data if no detail data was found for this candle return self._get_sell_trade_entry_for_candle(trade, sell_row) - detail_data['enter_long'] = sell_row[LONG_IDX] - detail_data['exit_long'] = sell_row[ELONG_IDX] - detail_data['enter_short'] = sell_row[SHORT_IDX] - detail_data['exit_short'] = sell_row[ESHORT_IDX] + detail_data.loc[:, 'enter_long'] = sell_row[LONG_IDX] + detail_data.loc[:, 'exit_long'] = sell_row[ELONG_IDX] + detail_data.loc[:, 'enter_long'] = sell_row[LONG_IDX] + detail_data.loc[:, 'exit_long'] = sell_row[ELONG_IDX] headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long', 'enter_short', 'exit_short'] for det_row in detail_data[headers].values.tolist(): diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index af904f693..8662fc36d 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -30,7 +30,8 @@ class PairLocks(): PairLocks.locks = [] @staticmethod - def lock_pair(pair: str, until: datetime, reason: str = None, *, now: datetime = None) -> None: + def lock_pair(pair: str, until: datetime, reason: str = None, *, + now: datetime = None) -> PairLock: """ Create PairLock from now to "until". Uses database by default, unless PairLocks.use_db is set to False, @@ -52,6 +53,7 @@ class PairLocks(): PairLock.query.session.commit() else: PairLocks.locks.append(lock) + return lock @staticmethod def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]: diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index f33e5b4bc..2510d6fee 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from typing import Dict, List, Optional from freqtrade.persistence import PairLocks +from freqtrade.persistence.models import PairLock from freqtrade.plugins.protections import IProtection from freqtrade.resolvers import ProtectionResolver @@ -43,30 +44,28 @@ class ProtectionManager(): """ return [{p.name: p.short_desc()} for p in self._protection_handlers] - def global_stop(self, now: Optional[datetime] = None) -> bool: + def global_stop(self, now: Optional[datetime] = None) -> Optional[PairLock]: if not now: now = datetime.now(timezone.utc) - result = False + result = None for protection_handler in self._protection_handlers: if protection_handler.has_global_stop: - result, until, reason = protection_handler.global_stop(now) + lock, until, reason = protection_handler.global_stop(now) # Early stopping - first positive result blocks further trades - if result and until: + if lock and until: if not PairLocks.is_global_lock(until): - PairLocks.lock_pair('*', until, reason, now=now) - result = True + result = PairLocks.lock_pair('*', until, reason, now=now) return result - def stop_per_pair(self, pair, now: Optional[datetime] = None) -> bool: + def stop_per_pair(self, pair, now: Optional[datetime] = None) -> Optional[PairLock]: if not now: now = datetime.now(timezone.utc) - result = False + result = None for protection_handler in self._protection_handlers: if protection_handler.has_local_stop: - result, until, reason = protection_handler.stop_per_pair(pair, now) - if result and until: + lock, until, reason = protection_handler.stop_per_pair(pair, now) + if lock and until: if not PairLocks.is_pair_locked(pair, until): - PairLocks.lock_pair(pair, until, reason, now=now) - result = True + result = PairLocks.lock_pair(pair, until, reason, now=now) return result diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 4623c187e..edbc39772 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -4,6 +4,7 @@ from copy import deepcopy from fastapi import APIRouter, BackgroundTasks, Depends +from freqtrade.configuration.config_validation import validate_config_consistency from freqtrade.enums import BacktestState from freqtrade.exceptions import DependencyException from freqtrade.rpc.api_server.api_schemas import BacktestRequest, BacktestResponse @@ -42,38 +43,40 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac # Reload strategy lastconfig = ApiServer._bt_last_config strat = StrategyResolver.load_strategy(btconfig) + validate_config_consistency(btconfig) if ( not ApiServer._bt or lastconfig.get('timeframe') != strat.timeframe or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail') - or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0) or lastconfig.get('timerange') != btconfig['timerange'] ): from freqtrade.optimize.backtesting import Backtesting ApiServer._bt = Backtesting(btconfig) if ApiServer._bt.timeframe_detail: ApiServer._bt.load_bt_data_detail() - + else: + ApiServer._bt.config = btconfig + ApiServer._bt.init_backtest() # Only reload data if timeframe changed. if ( not ApiServer._bt_data or not ApiServer._bt_timerange - or lastconfig.get('stake_amount') != btconfig.get('stake_amount') - or lastconfig.get('enable_protections') != btconfig.get('enable_protections') - or lastconfig.get('protections') != btconfig.get('protections', []) or lastconfig.get('timeframe') != strat.timeframe + or lastconfig.get('timerange') != btconfig['timerange'] ): - lastconfig['timerange'] = btconfig['timerange'] - lastconfig['protections'] = btconfig.get('protections', []) - lastconfig['enable_protections'] = btconfig.get('enable_protections') - lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') - lastconfig['timeframe'] = strat.timeframe ApiServer._bt_data, ApiServer._bt_timerange = ApiServer._bt.load_bt_data() + lastconfig['timerange'] = btconfig['timerange'] + lastconfig['timeframe'] = strat.timeframe + lastconfig['protections'] = btconfig.get('protections', []) + lastconfig['enable_protections'] = btconfig.get('enable_protections') + lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') + ApiServer._bt.abort = False min_date, max_date = ApiServer._bt.backtest_one_strategy( strat, ApiServer._bt_data, ApiServer._bt_timerange) + ApiServer._bt.results = generate_backtest_stats( ApiServer._bt_data, ApiServer._bt.all_results, min_date=min_date, max_date=max_date) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 19c58b63d..059ba9c41 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -260,6 +260,50 @@ class Telegram(RPCHandler): return message + def compose_message(self, msg: Dict[str, Any], msg_type: RPCMessageType) -> str: + + if msg_type == RPCMessageType.BUY: + message = self._format_buy_msg(msg) + + elif msg_type in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL): + msg['message_side'] = 'buy' if msg_type == RPCMessageType.BUY_CANCEL else 'sell' + message = ("\N{WARNING SIGN} *{exchange}:* " + "Cancelling open {message_side} Order for {pair} (#{trade_id}). " + "Reason: {reason}.".format(**msg)) + + elif msg_type == RPCMessageType.BUY_FILL: + message = ("\N{LARGE CIRCLE} *{exchange}:* " + "Buy order for {pair} (#{trade_id}) filled " + "for {open_rate}.".format(**msg)) + elif msg_type == RPCMessageType.SELL_FILL: + message = ("\N{LARGE CIRCLE} *{exchange}:* " + "Sell order for {pair} (#{trade_id}) filled " + "for {close_rate}.".format(**msg)) + elif msg_type == RPCMessageType.SELL: + message = self._format_sell_msg(msg) + elif msg_type == RPCMessageType.PROTECTION_TRIGGER: + message = ( + "*Protection* triggered due to {reason}. " + "`{pair}` will be locked until `{lock_end_time}`." + ).format(**msg) + elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL: + message = ( + "*Protection* triggered due to {reason}. " + "*All pairs* will be locked until `{lock_end_time}`." + ).format(**msg) + elif msg_type == RPCMessageType.STATUS: + message = '*Status:* `{status}`'.format(**msg) + + elif msg_type == RPCMessageType.WARNING: + message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg) + + elif msg_type == RPCMessageType.STARTUP: + message = '{status}'.format(**msg) + + else: + raise NotImplementedError('Unknown message type: {}'.format(msg_type)) + return message + def send_msg(self, msg: Dict[str, Any]) -> None: """ Send a message to telegram channel """ @@ -284,37 +328,7 @@ class Telegram(RPCHandler): # Notification disabled return - if msg_type == RPCMessageType.BUY: - message = self._format_buy_msg(msg) - - elif msg_type in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL): - msg['message_side'] = 'buy' if msg_type == RPCMessageType.BUY_CANCEL else 'sell' - message = ("\N{WARNING SIGN} *{exchange}:* " - "Cancelling open {message_side} Order for {pair} (#{trade_id}). " - "Reason: {reason}.".format(**msg)) - - elif msg_type == RPCMessageType.BUY_FILL: - message = ("\N{LARGE CIRCLE} *{exchange}:* " - "Buy order for {pair} (#{trade_id}) filled " - "for {open_rate}.".format(**msg)) - elif msg_type == RPCMessageType.SELL_FILL: - message = ("\N{LARGE CIRCLE} *{exchange}:* " - "Sell order for {pair} (#{trade_id}) filled " - "for {close_rate}.".format(**msg)) - elif msg_type == RPCMessageType.SELL: - message = self._format_sell_msg(msg) - - elif msg_type == RPCMessageType.STATUS: - message = '*Status:* `{status}`'.format(**msg) - - elif msg_type == RPCMessageType.WARNING: - message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg) - - elif msg_type == RPCMessageType.STARTUP: - message = '{status}'.format(**msg) - - else: - raise NotImplementedError('Unknown message type: {}'.format(msg_type)) + message = self.compose_message(msg, msg_type) self._send_msg(message, disable_notification=(noti == 'silent')) diff --git a/freqtrade/templates/subtemplates/exchange_kucoin.j2 b/freqtrade/templates/subtemplates/exchange_kucoin.j2 index f9dfff663..9882c51c7 100644 --- a/freqtrade/templates/subtemplates/exchange_kucoin.j2 +++ b/freqtrade/templates/subtemplates/exchange_kucoin.j2 @@ -4,7 +4,7 @@ "secret": "{{ exchange_secret }}", "password": "{{ exchange_key_password }}", "ccxt_config": { - "enableRateLimit": true + "enableRateLimit": true, "rateLimit": 200 }, "ccxt_async_config": { diff --git a/requirements-dev.txt b/requirements-dev.txt index 4859e1cc6..2f03255a0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,10 +18,10 @@ isort==5.9.3 time-machine==2.4.0 # Convert jupyter notebooks to markdown documents -nbconvert==6.1.0 +nbconvert==6.2.0 # mypy types types-cachetools==4.2.0 types-filelock==0.1.5 -types-requests==2.25.6 +types-requests==2.25.9 types-tabulate==0.8.2 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 7dc55a9fc..9feec80f1 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -8,4 +8,4 @@ scikit-optimize==0.8.1 filelock==3.0.12 joblib==1.0.1 psutil==5.8.0 -progressbar2==3.53.2 +progressbar2==3.53.3 diff --git a/requirements.txt b/requirements.txt index aa729dd9f..feeb4d942 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,17 @@ numpy==1.21.2 pandas==1.3.3 +pandas-ta==0.3.14b -ccxt==1.56.30 +ccxt==1.57.3 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.8 aiohttp==3.7.4.post0 -SQLAlchemy==1.4.23 +SQLAlchemy==1.4.25 python-telegram-bot==13.7 arrow==1.1.1 cachetools==4.2.2 requests==2.26.0 -urllib3==1.26.6 +urllib3==1.26.7 wrapt==1.12.1 jsonschema==3.2.0 TA-Lib==0.4.21 diff --git a/scripts/rest_client.py b/scripts/rest_client.py index ece0a253e..713b398c3 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -312,7 +312,7 @@ class FtRestClient(): :param limit: Limit result to the last n candles. :return: json object """ - return self._get("available_pairs", params={ + return self._get("pair_candles", params={ "pair": pair, "timeframe": timeframe, "limit": limit, diff --git a/setup.py b/setup.py index 727c40c7c..cf381bdd3 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ setup( 'wrapt', 'jsonschema', 'TA-Lib', + 'pandas-ta', 'technical', 'tabulate', 'pycoingecko', diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index a1d89d7d3..63b63a3b0 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -8,12 +8,12 @@ from zipfile import ZipFile import arrow import pytest -from freqtrade.commands import (start_convert_data, start_create_userdir, start_download_data, - start_hyperopt_list, start_hyperopt_show, start_install_ui, - start_list_data, start_list_exchanges, start_list_markets, - start_list_strategies, start_list_timeframes, start_new_strategy, - start_show_trades, start_test_pairlist, start_trading, - start_webserver) +from freqtrade.commands import (start_convert_data, start_convert_trades, start_create_userdir, + start_download_data, start_hyperopt_list, start_hyperopt_show, + start_install_ui, start_list_data, start_list_exchanges, + start_list_markets, start_list_strategies, start_list_timeframes, + start_new_strategy, start_show_trades, start_test_pairlist, + start_trading, start_webserver) from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) from freqtrade.configuration import setup_utils_configuration @@ -208,11 +208,10 @@ def test_list_timeframes(mocker, capsys): assert re.search(r"^1d$", captured.out, re.MULTILINE) -def test_list_markets(mocker, markets, capsys): +def test_list_markets(mocker, markets_static, capsys): api_mock = MagicMock() - api_mock.markets = markets - patch_exchange(mocker, api_mock=api_mock, id='bittrex') + patch_exchange(mocker, api_mock=api_mock, id='bittrex', mock_markets=markets_static) # Test with no --config args = [ @@ -237,7 +236,7 @@ def test_list_markets(mocker, markets, capsys): "TKN/BTC, XLTCUSDT, XRP/BTC.\n" in captured.out) - patch_exchange(mocker, api_mock=api_mock, id="binance") + patch_exchange(mocker, api_mock=api_mock, id="binance", mock_markets=markets_static) # Test with --exchange args = [ "list-markets", @@ -250,7 +249,7 @@ def test_list_markets(mocker, markets, capsys): assert re.match("\nExchange Binance has 10 active markets:\n", captured.out) - patch_exchange(mocker, api_mock=api_mock, id="bittrex") + patch_exchange(mocker, api_mock=api_mock, id="bittrex", mock_markets=markets_static) # Test with --all: all markets args = [ "list-markets", "--all", @@ -760,6 +759,22 @@ def test_download_data_trades(mocker, caplog): assert convert_mock.call_count == 1 +def test_start_convert_trades(mocker, caplog): + convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv', + MagicMock(return_value=[])) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + args = [ + "trades-to-ohlcv", + "--exchange", "kraken", + "--pairs", "ETH/BTC", "XRP/BTC", + ] + start_convert_trades(get_args(args)) + assert convert_mock.call_count == 1 + + def test_start_list_strategies(mocker, caplog, capsys): args = [ diff --git a/tests/conftest.py b/tests/conftest.py index b35ff17d6..eb978544f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -102,8 +102,10 @@ def patch_exchange( mocker.patch('freqtrade.exchange.Exchange.precisionMode', PropertyMock(return_value=2)) if mock_markets: + if isinstance(mock_markets, bool): + mock_markets = get_markets() mocker.patch('freqtrade.exchange.Exchange.markets', - PropertyMock(return_value=get_markets())) + PropertyMock(return_value=mock_markets)) if mock_supported_modes: mocker.patch( @@ -462,6 +464,8 @@ def markets(): def get_markets(): + # See get_markets_static() for immutable markets and do not modify them unless absolutely + # necessary! return { 'ETH/BTC': { 'id': 'ethbtc', @@ -778,11 +782,22 @@ def get_markets(): @pytest.fixture -def shitcoinmarkets(markets): +def markets_static(): + # These markets are used in some tests that would need adaptation should anything change in + # market list. Do not modify this list without a good reason! Do not modify market parameters + # of listed pairs in get_markets() without a good reason either! + static_markets = ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', + 'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XLTCUSDT', 'XRP/BTC'] + all_markets = get_markets() + return {m: all_markets[m] for m in static_markets} + + +@pytest.fixture +def shitcoinmarkets(markets_static): """ Fixture with shitcoin markets - used to test filters in pairlists """ - shitmarkets = deepcopy(markets) + shitmarkets = deepcopy(markets_static) shitmarkets.update({ 'HOT/BTC': { 'id': 'HOTBTC', @@ -1788,14 +1803,6 @@ def trades_for_order2(): 'fee': {'cost': 0.004, 'currency': 'LTC'}}] -@pytest.fixture(scope="function") -def trades_for_order3(trades_for_order2): - # Different fee currencies for each trade - trades_for_order = deepcopy(trades_for_order2) - trades_for_order[0]['fee'] = {'cost': 0.02, 'currency': 'BNB'} - return trades_for_order - - @pytest.fixture def buy_order_fee(): return { diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 8b16a9f12..38b8aba35 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2806,7 +2806,7 @@ def test_get_valid_pair_combination(default_conf, mocker, markets): (['LTC'], ['NONEXISTENT'], False, False, []), ]) -def test_get_markets(default_conf, mocker, markets, +def test_get_markets(default_conf, mocker, markets_static, base_currencies, quote_currencies, pairs_only, active_only, expected_keys): mocker.patch.multiple('freqtrade.exchange.Exchange', @@ -2814,7 +2814,7 @@ def test_get_markets(default_conf, mocker, markets, _load_async_markets=MagicMock(), validate_pairs=MagicMock(), validate_timeframes=MagicMock(), - markets=PropertyMock(return_value=markets)) + markets=PropertyMock(return_value=markets_static)) ex = Exchange(default_conf) pairs = ex.get_markets(base_currencies, quote_currencies, pairs_only, active_only) assert sorted(pairs.keys()) == sorted(expected_keys) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 1ce8d172c..cf918e2a0 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -131,9 +131,9 @@ def test_load_pairlist_noexist(mocker, markets, default_conf): default_conf, {}, 1) -def test_load_pairlist_verify_multi(mocker, markets, default_conf): +def test_load_pairlist_verify_multi(mocker, markets_static, default_conf): freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) + mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets_static)) plm = PairListManager(freqtrade.exchange, default_conf) # Call different versions one after the other, should always consider what was passed in # and have no side-effects (therefore the same check multiple times) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index c0a9ae72a..a3cb29c9d 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -125,7 +125,7 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): # Test 5m after lock-period - this should try and relock the pair, but end-time # should be the previous end-time end_time = PairLocks.get_pair_longest_lock('*').lock_end_time + timedelta(minutes=5) - assert freqtrade.protections.global_stop(end_time) + freqtrade.protections.global_stop(end_time) assert not PairLocks.is_global_lock(end_time) @@ -182,7 +182,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair min_ago_open=180, min_ago_close=30, profit_rate=0.9, )) - assert freqtrade.protections.stop_per_pair(pair) + freqtrade.protections.stop_per_pair(pair) assert freqtrade.protections.global_stop() != only_per_pair assert PairLocks.is_pair_locked(pair) assert PairLocks.is_global_lock() != only_per_pair diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 23ccadca0..791797ceb 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1313,6 +1313,34 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None: 'Reason: cancelled due to timeout.') +def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> None: + + default_conf['telegram']['notification_settings']['protection_trigger'] = 'on' + + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) + time_machine.move_to("2021-09-01 05:00:00 +00:00") + lock = PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=6).datetime, 'randreason') + msg = { + 'type': RPCMessageType.PROTECTION_TRIGGER, + } + msg.update(lock.to_json()) + telegram.send_msg(msg) + assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " + "`ETH/BTC` will be locked until `2021-09-01 05:10:00`.") + + msg_mock.reset_mock() + # Test global protection + + msg = { + 'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, + } + lock = PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=100).datetime, 'randreason') + msg.update(lock.to_json()) + telegram.send_msg(msg) + assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " + "*All pairs* will be locked until `2021-09-01 06:45:00`.") + + def test_send_msg_buy_fill_notification(default_conf, mocker) -> None: default_conf['telegram']['notification_settings']['buy_fill'] = 'on' diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 71926f9b7..a5bec30d9 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -416,6 +416,29 @@ def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order, assert log_has_re(message, caplog) +def test_handle_protections(mocker, default_conf, fee): + default_conf['protections'] = [ + {"method": "CooldownPeriod", "stop_duration": 60}, + { + "method": "StoplossGuard", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 4, + "only_per_pair": False + } + ] + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade.protections._protection_handlers[1].global_stop = MagicMock( + return_value=(True, arrow.utcnow().shift(hours=1).datetime, "asdf")) + create_mock_trades(fee) + freqtrade.handle_protections('ETC/BTC') + send_msg_mock = freqtrade.rpc.send_msg + assert send_msg_mock.call_count == 2 + assert send_msg_mock.call_args_list[0][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER + assert send_msg_mock.call_args_list[1][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER_GLOBAL + + def test_create_trade_no_signal(default_conf, fee, mocker) -> None: default_conf['dry_run'] = True @@ -3508,6 +3531,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe ) freqtrade = get_patched_freqtradebot(mocker, default_conf) + caplog.clear() # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has( @@ -3567,8 +3591,35 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f ) -def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fee, mocker): - trades_for_order[0]['fee']['currency'] = 'ETH' +@pytest.mark.parametrize( + 'fee_par,fee_reduction_amount,use_ticker_rate,expected_log', [ + # basic, amount does not change + ({'cost': 0.008, 'currency': 'ETH'}, 0, False, None), + # no currency in fee + ({'cost': 0.004, 'currency': None}, 0, True, None), + # BNB no rate + ({'cost': 0.00094518, 'currency': 'BNB'}, 0, True, ( + 'Fee for Trade Trade(id=None, pair=LTC/ETH, amount=8.00000000, is_short=False, ' + 'leverage=1.0, open_rate=0.24544100, open_since=closed) [buy]: 0.00094518 BNB -' + ' rate: None' + )), + # from order + ({'cost': 0.004, 'currency': 'LTC'}, 0.004, False, ( + 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + 'is_short=False, leverage=1.0, open_rate=0.24544100, open_since=closed) (from' + ' 8.0 to 7.996).' + )), + # invalid, no currency in from fee dict + ({'cost': 0.008, 'currency': None}, 0, True, None), + ]) +def test_get_real_amount( + default_conf, trades_for_order, buy_order_fee, fee, mocker, caplog, + fee_par, fee_reduction_amount, use_ticker_rate, expected_log +): + + buy_order = deepcopy(buy_order_fee) + buy_order['fee'] = fee_par + trades_for_order[0]['fee'] = fee_par mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) @@ -3583,19 +3634,38 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fe ) freqtrade = get_patched_freqtradebot(mocker, default_conf) - # Amount does not change - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount + if not use_ticker_rate: + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', side_effect=ExchangeError) + + caplog.clear() + assert freqtrade.get_real_amount(trade, buy_order) == amount - fee_reduction_amount + + if expected_log: + assert log_has(expected_log, caplog) -def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_order_fee, - fee, mocker): +@pytest.mark.parametrize( + 'fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount', [ + # basic, amount is reduced by fee + (None, None, 0.001, 0.001, 7.992), + # different fee currency on both trades, fee is average of both trade's fee + (0.02, 'BNB', 0.0005, 0.001518575, 7.996), + ]) +def test_get_real_amount_multi( + default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker, markets, + fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount, +): - limit_buy_order = deepcopy(buy_order_fee) - limit_buy_order['fee'] = {'cost': 0.004, 'currency': None} - trades_for_order[0]['fee']['currency'] = None + trades_for_order = deepcopy(trades_for_order2) + if fee_cost: + trades_for_order[0]['fee']['cost'] = fee_cost + if fee_currency: + trades_for_order[0]['fee']['currency'] = fee_currency mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - amount = sum(x['amount'] for x in trades_for_order) + amount = float(sum(x['amount'] for x in trades_for_order)) + default_conf['stake_currency'] = "ETH" + trade = Trade( pair='LTC/ETH', amount=amount, @@ -3605,78 +3675,7 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_ open_rate=0.245441, open_order_id="123456" ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - # Amount does not change - assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - - -def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, fee, mocker): - trades_for_order[0]['fee']['currency'] = 'BNB' - trades_for_order[0]['fee']['cost'] = 0.00094518 - - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - amount = sum(x['amount'] for x in trades_for_order) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" - ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - - # Amount does not change - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - - -def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker): - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order2) - amount = float(sum(x['amount'] for x in trades_for_order2)) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" - ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - - # Amount is reduced by "fee" - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) - assert log_has( - 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, is_short=False,' - ' leverage=1.0, open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).', - caplog - ) - - assert trade.fee_open == 0.001 - assert trade.fee_close == 0.001 - assert trade.fee_open_cost is not None - assert trade.fee_open_currency is not None - assert trade.fee_close_cost is None - assert trade.fee_close_currency is None - - -def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, caplog, fee, - mocker, markets): - # Different fee currency on both trades - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order3) - amount = float(sum(x['amount'] for x in trades_for_order3)) - default_conf['stake_currency'] = 'ETH' - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" - ) # Fake markets entry to enable fee parsing markets['BNB/ETH'] = markets['ETH/BTC'] freqtrade = get_patched_freqtradebot(mocker, default_conf) @@ -3685,50 +3684,25 @@ def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, return_value={'ask': 0.19, 'last': 0.2}) # Amount is reduced by "fee" - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.0005) + expected_amount = amount - (amount * fee_reduction_amount) + assert freqtrade.get_real_amount(trade, buy_order_fee) == expected_amount assert log_has( - 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, is_short=False,' - ' leverage=1.0, open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).', + ( + 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + 'is_short=False, leverage=1.0, open_rate=0.24544100, open_since=closed) ' + f'(from 8.0 to {expected_log_amount}).' + ), caplog ) - # Overall fee is average of both trade's fee - assert trade.fee_open == 0.001518575 + + assert trade.fee_open == expected_fee + assert trade.fee_close == expected_fee assert trade.fee_open_cost is not None assert trade.fee_open_currency is not None assert trade.fee_close_cost is None assert trade.fee_close_currency is None -def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, fee, - caplog, mocker): - limit_buy_order = deepcopy(buy_order_fee) - limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'} - - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', - return_value=[trades_for_order]) - amount = float(sum(x['amount'] for x in trades_for_order)) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" - ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - # Ticker rate cannot be found for this to work. - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', side_effect=ExchangeError) - - # Amount is reduced by "fee" - assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004 - assert log_has( - 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, is_short=False,' - ' leverage=1.0, open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).', - caplog - ) - - def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, fee, mocker): limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004} @@ -3796,27 +3770,6 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b abs_tol=MATH_CLOSE_PREC,) -def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, fee, mocker): - # Remove "Currency" from fee dict - trades_for_order[0]['fee'] = {'cost': 0.008} - - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - amount = sum(x['amount'] for x in trades_for_order) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - open_rate=0.245441, - fee_open=fee.return_value, - fee_close=fee.return_value, - - open_order_id="123456" - ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - # Amount does not change - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - - def test_get_real_amount_open_trade(default_conf, fee, mocker): amount = 12345 trade = Trade( @@ -3867,10 +3820,14 @@ def test_apply_fee_conditional(default_conf, fee, caplog, mocker, assert walletmock.call_count == 1 +@pytest.mark.parametrize("delta, is_high_delta", [ + (0.1, False), + (100, True), +]) def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open, limit_buy_order, - fee, mocker, order_book_l2): + fee, mocker, order_book_l2, delta, is_high_delta): default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True - default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 + default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = delta patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) @@ -3888,42 +3845,22 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open, freqtrade.enter_positions() trade = Trade.query.first() - assert trade is not None - assert trade.stake_amount == 0.001 - assert trade.is_open - assert trade.open_date is not None - assert trade.exchange == 'binance' + if is_high_delta: + assert trade is None + else: + assert trade is not None + assert trade.stake_amount == 0.001 + assert trade.is_open + assert trade.open_date is not None + assert trade.exchange == 'binance' - assert len(Trade.query.all()) == 1 + assert len(Trade.query.all()) == 1 - # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) - assert trade.open_rate == 0.00001099 - assert whitelist == default_conf['exchange']['pair_whitelist'] - - -def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_order, - fee, mocker, order_book_l2): - default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True - # delta is 100 which is impossible to reach. hence check_depth_of_market will return false - default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100 - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, - create_order=MagicMock(return_value={'id': limit_buy_order['id']}), - get_fee=fee, - ) - # Save state of current whitelist - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - freqtrade.enter_positions() - - trade = Trade.query.first() - assert trade is None + assert trade.open_rate == 0.00001099 + assert whitelist == default_conf['exchange']['pair_whitelist'] @pytest.mark.parametrize('exception_thrown,ask,last,order_book_top,order_book', [ @@ -4122,16 +4059,16 @@ def test_check_for_open_trades(mocker, default_conf, fee): @pytest.mark.usefixtures("init_persistence") -def test_update_open_orders(mocker, default_conf, fee, caplog): +def test_startup_update_open_orders(mocker, default_conf, fee, caplog): freqtrade = get_patched_freqtradebot(mocker, default_conf) create_mock_trades(fee) - freqtrade.update_open_orders() + freqtrade.startup_update_open_orders() assert not log_has_re(r"Error updating Order .*", caplog) caplog.clear() freqtrade.config['dry_run'] = False - freqtrade.update_open_orders() + freqtrade.startup_update_open_orders() assert log_has_re(r"Error updating Order .*", caplog) caplog.clear() @@ -4142,7 +4079,7 @@ def test_update_open_orders(mocker, default_conf, fee, caplog): 'status': 'closed', }) mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=matching_buy_order) - freqtrade.update_open_orders() + freqtrade.startup_update_open_orders() # Only stoploss and sell orders are kept open assert len(Order.get_open_orders()) == 2