Merge branch 'feat/short' into lev-freqtradebot

config_examples/config_full.example.json

@@ -149,7 +149,9 @@
             },
             "sell_fill": "on",
             "buy_cancel": "on",
-            "sell_cancel": "on"
+            "sell_cancel": "on",
+            "protection_trigger": "off",
+            "protection_trigger_global": "on"
         },
         "reload": true,
         "balance_dust_level": 0.01

docs/data-download.md

@@ -204,6 +204,61 @@ It'll also remove original jsongz data files (`--erase` parameter).
 freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase
 ```
 
+### Sub-command trades to ohlcv
+
+When you need to use `--dl-trades` (kraken only) to download data, conversion of trades data to ohlcv data is the last step.
+This command will allow you to repeat this last step for additional timeframes without re-downloading the data.
+
+```
+usage: freqtrade trades-to-ohlcv [-h] [-v] [--logfile FILE] [-V] [-c PATH]
+                                 [-d PATH] [--userdir PATH]
+                                 [-p PAIRS [PAIRS ...]]
+                                 [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
+                                 [--exchange EXCHANGE]
+                                 [--data-format-ohlcv {json,jsongz,hdf5}]
+                                 [--data-format-trades {json,jsongz,hdf5}]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
+                        Limit command to these pairs. Pairs are space-
+                        separated.
+  -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]
+                        Specify which tickers to download. Space-separated
+                        list. Default: `1m 5m`.
+  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
+                        config is provided.
+  --data-format-ohlcv {json,jsongz,hdf5}
+                        Storage format for downloaded candle (OHLCV) data.
+                        (default: `json`).
+  --data-format-trades {json,jsongz,hdf5}
+                        Storage format for downloaded trades data. (default:
+                        `jsongz`).
+
+Common arguments:
+  -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
+  --logfile FILE        Log to the file specified. Special values are:
+                        'syslog', 'journald'. See the documentation for more
+                        details.
+  -V, --version         show program's version number and exit
+  -c PATH, --config PATH
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
+  -d PATH, --datadir PATH
+                        Path to directory with historical backtesting data.
+  --userdir PATH, --user-data-dir PATH
+                        Path to userdata directory.
+
+```
+
+#### Example trade-to-ohlcv conversion
+
+``` bash
+freqtrade trades-to-ohlcv --exchange kraken -t 5m 1h 1d --pairs BTC/EUR ETH/EUR
+```
+
 ### Sub-command list-data
 
 You can get a list of downloaded data using the `list-data` sub-command.

docs/docker_quickstart.md

@@ -149,6 +149,24 @@ You'll then also need to modify the `docker-compose.yml` file and uncomment the
 
 You can then run `docker-compose build` to build the docker image, and run it using the commands described above.
 
+### Troubleshooting
+
+#### Docker on Windows
+
+* Error: `"Timestamp for this request is outside of the recvWindow."`
+  * The market api requests require a synchronized clock but the time in the docker container shifts a bit over time into the past.
+    To fix this issue temporarily you need to run `wsl --shutdown` and restart docker again (a popup on windows 10 will ask you to do so).
+    A permanent solution is either to host the docker container on a linux host or restart the wsl from time to time with the scheduler.
+    ```
+    taskkill /IM "Docker Desktop.exe" /F
+    wsl --shutdown
+    start "" "C:\Program Files\Docker\Docker\Docker Desktop.exe"
+    ```
+
+!!! Warning
+    Due to the above, we do not recommend the usage of docker on windows for production setups, but only for experimentation, datadownload and backtesting.
+    Best use a linux-VPS for running freqtrade reliably.
+
 ## Plotting with docker-compose
 
 Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your docker-compose.yml file.

docs/requirements-docs.txt

@@ -1,4 +1,4 @@
 mkdocs==1.2.2
-mkdocs-material==7.2.6
+mkdocs-material==7.3.0
 mdx_truly_sane_lists==1.2
 pymdown-extensions==8.2

docs/strategy-advanced.md

@@ -731,3 +731,33 @@ The variable 'content', will contain the strategy file in a BASE64 encoded form.
 ```
 
 Please ensure that 'NameOfStrategy' is identical to the strategy name!
+
+## Performance warning
+
+When executing a strategy, one can sometimes be greeted by the following in the logs
+
+> PerformanceWarning: DataFrame is highly fragmented.
+
+This is a warning from [`pandas`](https://github.com/pandas-dev/pandas) and as the warning continues to say:
+use `pd.concat(axis=1)`.
+This can have slight performance implications, which are usually only visible during hyperopt (when optimizing an indicator).
+
+For example:
+
+```python
+for val in self.buy_ema_short.range:
+    dataframe[f'ema_short_{val}'] = ta.EMA(dataframe, timeperiod=val)
+```
+
+should be rewritten to
+
+```python
+frames = [dataframe]
+for val in self.buy_ema_short.range:
+    frames.append({
+        f'ema_short_{val}': ta.EMA(dataframe, timeperiod=val)
+    })
+
+# Append columns to existing dataframe
+merged_frame = pd.concat(frames, axis=1)
+```

docs/strategy-customization.md

@@ -122,6 +122,16 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame
     Look into the [user_data/strategies/sample_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_strategy.py).
     Then uncomment indicators you need.
 
+#### Indicator libraries
+
+Out of the box, freqtrade installs the following technical libraries:
+
+* [ta-lib](http://mrjbq7.github.io/ta-lib/)
+* [pandas-ta](https://twopirllc.github.io/pandas-ta/)
+* [technical](https://github.com/freqtrade/technical/)
+
+Additional technical libraries can be installed as necessary, or custom indicators may be written / invented by the strategy author.
+
 ### Strategy startup period
 
 Most indicators have an instable startup period, in which they are either not available, or the calculation is incorrect. This can lead to inconsistencies, since Freqtrade does not know how long this instable period should be.
@@ -942,6 +952,8 @@ Printing more than a few rows is also possible (simply use  `print(dataframe)` i
 
 ## Common mistakes when developing strategies
 
+### Peeking into the future while backtesting
+
 Backtesting analyzes the whole time-range at once for performance reasons. Because of this, strategy authors need to make sure that strategies do not look-ahead into the future.
 This is a common pain-point, which can cause huge differences between backtesting and dry/live run methods, since they all use data which is not available during dry/live runs, so these strategies will perform well during backtesting, but will fail / perform badly in real conditions.
 

docs/telegram-usage.md

@@ -93,7 +93,9 @@ Example configuration showing the different settings:
         "buy_cancel": "silent",
         "sell_cancel": "on",
         "buy_fill": "off",
-        "sell_fill": "off"
+        "sell_fill": "off",
+        "protection_trigger": "off",
+        "protection_trigger_global": "on"
     },
     "reload": true,
     "balance_dust_level": 0.01
@@ -103,6 +105,7 @@ Example configuration showing the different settings:
 `buy` notifications are sent when the order is placed, while `buy_fill` notifications are sent when the order is filled on the exchange.
 `sell` notifications are sent when the order is placed, while `sell_fill` notifications are sent when the order is filled on the exchange.
 `*_fill` notifications are off by default and must be explicitly enabled.
+`protection_trigger` notifications are sent when a protection triggers and `protection_trigger_global` notifications trigger when global protections are triggered.
 
 
 `balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown.

freqtrade/commands/__init__.py

@@ -8,8 +8,8 @@ Note: Be careful with file-scoped imports in these subfiles.
 """
 from freqtrade.commands.arguments import Arguments
 from freqtrade.commands.build_config_commands import start_new_config
-from freqtrade.commands.data_commands import (start_convert_data, start_download_data,
+from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades,
-                                              start_list_data)
+                                              start_download_data, start_list_data)
 from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui,
                                                 start_new_strategy)
 from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show

freqtrade/commands/arguments.py

@@ -58,6 +58,8 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"]
 ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"]
 ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
 
+ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"]
+
 ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
 
 ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange",
@@ -91,7 +93,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop
 NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
                     "list-markets", "list-pairs", "list-strategies", "list-data",
                     "hyperopt-list", "hyperopt-show",
-                    "plot-dataframe", "plot-profit", "show-trades"]
+                    "plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv"]
 
 NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]
 
@@ -169,14 +171,14 @@ class Arguments:
         self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot')
         self._build_args(optionlist=['version'], parser=self.parser)
 
-        from freqtrade.commands import (start_backtesting, start_convert_data, start_create_userdir,
+        from freqtrade.commands import (start_backtesting, start_convert_data, start_convert_trades,
-                                        start_download_data, start_edge, start_hyperopt,
+                                        start_create_userdir, start_download_data, start_edge,
-                                        start_hyperopt_list, start_hyperopt_show, start_install_ui,
+                                        start_hyperopt, start_hyperopt_list, start_hyperopt_show,
-                                        start_list_data, start_list_exchanges, start_list_markets,
+                                        start_install_ui, start_list_data, start_list_exchanges,
-                                        start_list_strategies, start_list_timeframes,
+                                        start_list_markets, start_list_strategies,
-                                        start_new_config, start_new_strategy, start_plot_dataframe,
+                                        start_list_timeframes, start_new_config, start_new_strategy,
-                                        start_plot_profit, start_show_trades, start_test_pairlist,
+                                        start_plot_dataframe, start_plot_profit, start_show_trades,
-                                        start_trading, start_webserver)
+                                        start_test_pairlist, start_trading, start_webserver)
 
         subparsers = self.parser.add_subparsers(dest='command',
                                                 # Use custom message when no subhandler is added
@@ -236,6 +238,15 @@ class Arguments:
         convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False))
         self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd)
 
+        # Add trades-to-ohlcv subcommand
+        convert_trade_data_cmd = subparsers.add_parser(
+            'trades-to-ohlcv',
+            help='Convert trade data to OHLCV data.',
+            parents=[_common_parser],
+        )
+        convert_trade_data_cmd.set_defaults(func=start_convert_trades)
+        self._build_args(optionlist=ARGS_CONVERT_TRADES, parser=convert_trade_data_cmd)
+
         # Add list-data subcommand
         list_data_cmd = subparsers.add_parser(
             'list-data',

freqtrade/commands/cli_options.py

@@ -381,12 +381,12 @@ AVAILABLE_CLI_OPTIONS = {
     ),
     "dataformat_ohlcv": Arg(
         '--data-format-ohlcv',
-        help='Storage format for downloaded candle (OHLCV) data. (default: `%(default)s`).',
+        help='Storage format for downloaded candle (OHLCV) data. (default: `json`).',
         choices=constants.AVAILABLE_DATAHANDLERS,
     ),
     "dataformat_trades": Arg(
         '--data-format-trades',
-        help='Storage format for downloaded trades data. (default: `%(default)s`).',
+        help='Storage format for downloaded trades data. (default: `jsongz`).',
         choices=constants.AVAILABLE_DATAHANDLERS,
     ),
     "exchange": Arg(

freqtrade/commands/data_commands.py

@@ -89,6 +89,41 @@ def start_download_data(args: Dict[str, Any]) -> None:
                         f"on exchange {exchange.name}.")
 
 
+def start_convert_trades(args: Dict[str, Any]) -> None:
+
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
+
+    timerange = TimeRange()
+
+    # Remove stake-currency to skip checks which are not relevant for datadownload
+    config['stake_currency'] = ''
+
+    if 'pairs' not in config:
+        raise OperationalException(
+            "Downloading data requires a list of pairs. "
+            "Please check the documentation on how to configure this.")
+
+    # Init exchange
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
+    # Manual validations of relevant settings
+    if not config['exchange'].get('skip_pair_validation', False):
+        exchange.validate_pairs(config['pairs'])
+    expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
+
+    logger.info(f"About to Convert pairs: {expanded_pairs}, "
+                f"intervals: {config['timeframes']} to {config['datadir']}")
+
+    for timeframe in config['timeframes']:
+        exchange.validate_timeframes(timeframe)
+    # Convert downloaded trade data to different timeframes
+    convert_trades_to_ohlcv(
+        pairs=expanded_pairs, timeframes=config['timeframes'],
+        datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
+        data_format_ohlcv=config['dataformat_ohlcv'],
+        data_format_trades=config['dataformat_trades'],
+    )
+
+
 def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
     """
     Convert data from one format to another

freqtrade/constants.py

@@ -110,7 +110,7 @@ CONF_SCHEMA = {
         },
         'tradable_balance_ratio': {
             'type': 'number',
-            'minimum': 0.1,
+            'minimum': 0.0,
             'maximum': 1,
             'default': 0.99
         },
@@ -284,6 +284,15 @@ CONF_SCHEMA = {
                             'enum': TELEGRAM_SETTING_OPTIONS,
                             'default': 'off'
                         },
+                        'protection_trigger': {
+                            'type': 'string',
+                            'enum': TELEGRAM_SETTING_OPTIONS,
+                            'default': 'off'
+                        },
+                        'protection_trigger_global': {
+                            'type': 'string',
+                            'enum': TELEGRAM_SETTING_OPTIONS,
+                        },
                     }
                 },
                 'reload': {'type': 'boolean'},

freqtrade/data/dataprovider.py

@@ -149,6 +149,8 @@ class DataProvider:
         Clear pair dataframe cache.
         """
         self.__cached_pairs = {}
+        self.__cached_pairs_backtesting = {}
+        self.__slice_index = 0
 
     # Exchange functions
 

freqtrade/enums/rpcmessagetype.py

@@ -13,6 +13,8 @@ class RPCMessageType(Enum):
     SELL = 'sell'
     SELL_FILL = 'sell_fill'
     SELL_CANCEL = 'sell_cancel'
+    PROTECTION_TRIGGER = 'protection_trigger'
+    PROTECTION_TRIGGER_GLOBAL = 'protection_trigger_global'
 
     SHORT = 'short'
     SHORT_FILL = 'short_fill'

freqtrade/freqtradebot.py

@@ -154,7 +154,7 @@ class FreqtradeBot(LoggingMixin):
 
         # Only update open orders on startup
         # This will update the database after the initial migration
-        self.update_open_orders()
+        self.startup_update_open_orders()
 
     def process(self) -> None:
         """
@@ -252,7 +252,7 @@ class FreqtradeBot(LoggingMixin):
         open_trades = len(Trade.get_open_trades())
         return max(0, self.config['max_open_trades'] - open_trades)
 
-    def update_open_orders(self):
+    def startup_update_open_orders(self):
         """
         Updates open orders based on order list kept in the database.
         Mainly updates the state of orders - but may also close trades
@@ -1380,7 +1380,7 @@ class FreqtradeBot(LoggingMixin):
             'exchange': trade.exchange.capitalize(),
             'pair': trade.pair,
             'gain': gain,
-            'limit': profit_rate,
+            'limit': profit_rate or 0,
             'order_type': order_type,
             'amount': trade.amount,
             'open_rate': trade.open_rate,
@@ -1389,7 +1389,7 @@ class FreqtradeBot(LoggingMixin):
             'profit_ratio': profit_ratio,
             'sell_reason': trade.sell_reason,
             'open_date': trade.open_date,
-            'close_date': trade.close_date,
+            'close_date': trade.close_date or datetime.now(timezone.utc),
             'stake_currency': self.config['stake_currency'],
             'fiat_currency': self.config.get('fiat_display_currency', None),
             'reason': reason,
@@ -1455,8 +1455,7 @@ class FreqtradeBot(LoggingMixin):
         if not trade.is_open:
             if not stoploss_order and not trade.open_order_id:
                 self._notify_exit(trade, '', True)
-            self.protections.stop_per_pair(trade.pair)
+            self.handle_protections(trade.pair)
-            self.protections.global_stop()
             self.wallets.update()
         elif not trade.open_order_id:
             # Buy fill
@@ -1464,6 +1463,19 @@ class FreqtradeBot(LoggingMixin):
 
         return False
 
+    def handle_protections(self, pair: str) -> None:
+        prot_trig = self.protections.stop_per_pair(pair)
+        if prot_trig:
+            msg = {'type': RPCMessageType.PROTECTION_TRIGGER, }
+            msg.update(prot_trig.to_json())
+            self.rpc.send_msg(msg)
+
+        prot_trig_glb = self.protections.global_stop()
+        if prot_trig_glb:
+            msg = {'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, }
+            msg.update(prot_trig_glb.to_json())
+            self.rpc.send_msg(msg)
+
     def apply_fee_conditional(self, trade: Trade, trade_base_currency: str,
                               amount: float, fee_abs: float) -> float:
         """

freqtrade/optimize/backtesting.py

@@ -87,18 +87,7 @@ class Backtesting:
                                        "configuration or as cli argument `--timeframe 5m`")
         self.timeframe = str(self.config.get('timeframe'))
         self.timeframe_min = timeframe_to_minutes(self.timeframe)
-        # Load detail timeframe if specified
+        self.init_backtest_detail()
-        self.timeframe_detail = str(self.config.get('timeframe_detail', ''))
-        if self.timeframe_detail:
-            self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail)
-            if self.timeframe_min <= self.timeframe_detail_min:
-                raise OperationalException(
-                    "Detail timeframe must be smaller than strategy timeframe.")
-
-        else:
-            self.timeframe_detail_min = 0
-        self.detail_data: Dict[str, DataFrame] = {}
-
         self.pairlists = PairListManager(self.exchange, self.config)
         if 'VolumePairList' in self.pairlists.name_list:
             raise OperationalException("VolumePairList not allowed for backtesting.")
@@ -121,14 +110,6 @@ class Backtesting:
         else:
             self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0])
 
-        Trade.use_db = False
-        Trade.reset_trades()
-        PairLocks.timeframe = self.config['timeframe']
-        PairLocks.use_db = False
-        PairLocks.reset_locks()
-
-        self.wallets = Wallets(self.config, self.exchange, log=False)
-
         self.timerange = TimeRange.parse_timerange(
             None if self.config.get('timerange') is None else str(self.config.get('timerange')))
 
@@ -144,6 +125,7 @@ class Backtesting:
 
         self.progress = BTProgress()
         self.abort = False
+        self.init_backtest()
 
     def __del__(self):
         self.cleanup()
@@ -153,6 +135,28 @@ class Backtesting:
         PairLocks.use_db = True
         Trade.use_db = True
 
+    def init_backtest_detail(self):
+        # Load detail timeframe if specified
+        self.timeframe_detail = str(self.config.get('timeframe_detail', ''))
+        if self.timeframe_detail:
+            self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail)
+            if self.timeframe_min <= self.timeframe_detail_min:
+                raise OperationalException(
+                    "Detail timeframe must be smaller than strategy timeframe.")
+
+        else:
+            self.timeframe_detail_min = 0
+        self.detail_data: Dict[str, DataFrame] = {}
+
+    def init_backtest(self):
+
+        self.prepare_backtest(False)
+
+        self.wallets = Wallets(self.config, self.exchange, log=False)
+
+        self.progress = BTProgress()
+        self.abort = False
+
     def _set_strategy(self, strategy: IStrategy):
         """
         Load strategy into backtesting
@@ -232,6 +236,7 @@ class Backtesting:
         Trade.reset_trades()
         self.rejected_trades = 0
         self.dataprovider.clear_cache()
+        if enable_protections:
             self._load_protections(self.strategy)
 
     def check_abort(self):
@@ -397,14 +402,14 @@ class Backtesting:
             detail_data = detail_data.loc[
                 (detail_data['date'] >= sell_candle_time) &
                 (detail_data['date'] < sell_candle_end)
-            ]
+            ].copy()
             if len(detail_data) == 0:
                 # Fall back to "regular" data if no detail data was found for this candle
                 return self._get_sell_trade_entry_for_candle(trade, sell_row)
-            detail_data['enter_long'] = sell_row[LONG_IDX]
+            detail_data.loc[:, 'enter_long'] = sell_row[LONG_IDX]
-            detail_data['exit_long'] = sell_row[ELONG_IDX]
+            detail_data.loc[:, 'exit_long'] = sell_row[ELONG_IDX]
-            detail_data['enter_short'] = sell_row[SHORT_IDX]
+            detail_data.loc[:, 'enter_long'] = sell_row[LONG_IDX]
-            detail_data['exit_short'] = sell_row[ESHORT_IDX]
+            detail_data.loc[:, 'exit_long'] = sell_row[ELONG_IDX]
             headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long',
                        'enter_short', 'exit_short']
             for det_row in detail_data[headers].values.tolist():

freqtrade/persistence/pairlock_middleware.py

@@ -30,7 +30,8 @@ class PairLocks():
             PairLocks.locks = []
 
     @staticmethod
-    def lock_pair(pair: str, until: datetime, reason: str = None, *, now: datetime = None) -> None:
+    def lock_pair(pair: str, until: datetime, reason: str = None, *,
+                  now: datetime = None) -> PairLock:
         """
         Create PairLock from now to "until".
         Uses database by default, unless PairLocks.use_db is set to False,
@@ -52,6 +53,7 @@ class PairLocks():
             PairLock.query.session.commit()
         else:
             PairLocks.locks.append(lock)
+        return lock
 
     @staticmethod
     def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]:

freqtrade/plugins/protectionmanager.py

@@ -6,6 +6,7 @@ from datetime import datetime, timezone
 from typing import Dict, List, Optional
 
 from freqtrade.persistence import PairLocks
+from freqtrade.persistence.models import PairLock
 from freqtrade.plugins.protections import IProtection
 from freqtrade.resolvers import ProtectionResolver
 
@@ -43,30 +44,28 @@ class ProtectionManager():
         """
         return [{p.name: p.short_desc()} for p in self._protection_handlers]
 
-    def global_stop(self, now: Optional[datetime] = None) -> bool:
+    def global_stop(self, now: Optional[datetime] = None) -> Optional[PairLock]:
         if not now:
             now = datetime.now(timezone.utc)
-        result = False
+        result = None
         for protection_handler in self._protection_handlers:
             if protection_handler.has_global_stop:
-                result, until, reason = protection_handler.global_stop(now)
+                lock, until, reason = protection_handler.global_stop(now)
 
                 # Early stopping - first positive result blocks further trades
-                if result and until:
+                if lock and until:
                     if not PairLocks.is_global_lock(until):
-                        PairLocks.lock_pair('*', until, reason, now=now)
+                        result = PairLocks.lock_pair('*', until, reason, now=now)
-                    result = True
         return result
 
-    def stop_per_pair(self, pair, now: Optional[datetime] = None) -> bool:
+    def stop_per_pair(self, pair, now: Optional[datetime] = None) -> Optional[PairLock]:
         if not now:
             now = datetime.now(timezone.utc)
-        result = False
+        result = None
         for protection_handler in self._protection_handlers:
             if protection_handler.has_local_stop:
-                result, until, reason = protection_handler.stop_per_pair(pair, now)
+                lock, until, reason = protection_handler.stop_per_pair(pair, now)
-                if result and until:
+                if lock and until:
                     if not PairLocks.is_pair_locked(pair, until):
-                        PairLocks.lock_pair(pair, until, reason, now=now)
+                        result = PairLocks.lock_pair(pair, until, reason, now=now)
-                    result = True
         return result

freqtrade/rpc/api_server/api_backtest.py

@@ -4,6 +4,7 @@ from copy import deepcopy
 
 from fastapi import APIRouter, BackgroundTasks, Depends
 
+from freqtrade.configuration.config_validation import validate_config_consistency
 from freqtrade.enums import BacktestState
 from freqtrade.exceptions import DependencyException
 from freqtrade.rpc.api_server.api_schemas import BacktestRequest, BacktestResponse
@@ -42,38 +43,40 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
             # Reload strategy
             lastconfig = ApiServer._bt_last_config
             strat = StrategyResolver.load_strategy(btconfig)
+            validate_config_consistency(btconfig)
 
             if (
                 not ApiServer._bt
                 or lastconfig.get('timeframe') != strat.timeframe
                 or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail')
-                or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0)
                 or lastconfig.get('timerange') != btconfig['timerange']
             ):
                 from freqtrade.optimize.backtesting import Backtesting
                 ApiServer._bt = Backtesting(btconfig)
                 if ApiServer._bt.timeframe_detail:
                     ApiServer._bt.load_bt_data_detail()
-
+            else:
+                ApiServer._bt.config = btconfig
+                ApiServer._bt.init_backtest()
             # Only reload data if timeframe changed.
             if (
                 not ApiServer._bt_data
                 or not ApiServer._bt_timerange
-                or lastconfig.get('stake_amount') != btconfig.get('stake_amount')
-                or lastconfig.get('enable_protections') != btconfig.get('enable_protections')
-                or lastconfig.get('protections') != btconfig.get('protections', [])
                 or lastconfig.get('timeframe') != strat.timeframe
+                or lastconfig.get('timerange') != btconfig['timerange']
             ):
+                ApiServer._bt_data, ApiServer._bt_timerange = ApiServer._bt.load_bt_data()
+
             lastconfig['timerange'] = btconfig['timerange']
+            lastconfig['timeframe'] = strat.timeframe
             lastconfig['protections'] = btconfig.get('protections', [])
             lastconfig['enable_protections'] = btconfig.get('enable_protections')
             lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet')
-                lastconfig['timeframe'] = strat.timeframe
-                ApiServer._bt_data, ApiServer._bt_timerange = ApiServer._bt.load_bt_data()
 
             ApiServer._bt.abort = False
             min_date, max_date = ApiServer._bt.backtest_one_strategy(
                 strat, ApiServer._bt_data, ApiServer._bt_timerange)
+
             ApiServer._bt.results = generate_backtest_stats(
                 ApiServer._bt_data, ApiServer._bt.all_results,
                 min_date=min_date, max_date=max_date)

freqtrade/rpc/telegram.py

@@ -260,6 +260,50 @@ class Telegram(RPCHandler):
 
         return message
 
+    def compose_message(self, msg: Dict[str, Any], msg_type: RPCMessageType) -> str:
+
+        if msg_type == RPCMessageType.BUY:
+            message = self._format_buy_msg(msg)
+
+        elif msg_type in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL):
+            msg['message_side'] = 'buy' if msg_type == RPCMessageType.BUY_CANCEL else 'sell'
+            message = ("\N{WARNING SIGN} *{exchange}:* "
+                       "Cancelling open {message_side} Order for {pair} (#{trade_id}). "
+                       "Reason: {reason}.".format(**msg))
+
+        elif msg_type == RPCMessageType.BUY_FILL:
+            message = ("\N{LARGE CIRCLE} *{exchange}:* "
+                       "Buy order for {pair} (#{trade_id}) filled "
+                       "for {open_rate}.".format(**msg))
+        elif msg_type == RPCMessageType.SELL_FILL:
+            message = ("\N{LARGE CIRCLE} *{exchange}:* "
+                       "Sell order for {pair} (#{trade_id}) filled "
+                       "for {close_rate}.".format(**msg))
+        elif msg_type == RPCMessageType.SELL:
+            message = self._format_sell_msg(msg)
+        elif msg_type == RPCMessageType.PROTECTION_TRIGGER:
+            message = (
+                "*Protection* triggered due to {reason}. "
+                "`{pair}` will be locked until `{lock_end_time}`."
+            ).format(**msg)
+        elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL:
+            message = (
+                "*Protection* triggered due to {reason}. "
+                "*All pairs* will be locked until `{lock_end_time}`."
+            ).format(**msg)
+        elif msg_type == RPCMessageType.STATUS:
+            message = '*Status:* `{status}`'.format(**msg)
+
+        elif msg_type == RPCMessageType.WARNING:
+            message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg)
+
+        elif msg_type == RPCMessageType.STARTUP:
+            message = '{status}'.format(**msg)
+
+        else:
+            raise NotImplementedError('Unknown message type: {}'.format(msg_type))
+        return message
+
     def send_msg(self, msg: Dict[str, Any]) -> None:
         """ Send a message to telegram channel """
 
@@ -284,37 +328,7 @@ class Telegram(RPCHandler):
             # Notification disabled
             return
 
-        if msg_type == RPCMessageType.BUY:
+        message = self.compose_message(msg, msg_type)
-            message = self._format_buy_msg(msg)
-
-        elif msg_type in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL):
-            msg['message_side'] = 'buy' if msg_type == RPCMessageType.BUY_CANCEL else 'sell'
-            message = ("\N{WARNING SIGN} *{exchange}:* "
-                       "Cancelling open {message_side} Order for {pair} (#{trade_id}). "
-                       "Reason: {reason}.".format(**msg))
-
-        elif msg_type == RPCMessageType.BUY_FILL:
-            message = ("\N{LARGE CIRCLE} *{exchange}:* "
-                       "Buy order for {pair} (#{trade_id}) filled "
-                       "for {open_rate}.".format(**msg))
-        elif msg_type == RPCMessageType.SELL_FILL:
-            message = ("\N{LARGE CIRCLE} *{exchange}:* "
-                       "Sell order for {pair} (#{trade_id}) filled "
-                       "for {close_rate}.".format(**msg))
-        elif msg_type == RPCMessageType.SELL:
-            message = self._format_sell_msg(msg)
-
-        elif msg_type == RPCMessageType.STATUS:
-            message = '*Status:* `{status}`'.format(**msg)
-
-        elif msg_type == RPCMessageType.WARNING:
-            message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg)
-
-        elif msg_type == RPCMessageType.STARTUP:
-            message = '{status}'.format(**msg)
-
-        else:
-            raise NotImplementedError('Unknown message type: {}'.format(msg_type))
 
         self._send_msg(message, disable_notification=(noti == 'silent'))
 

freqtrade/templates/subtemplates/exchange_kucoin.j2

@@ -4,7 +4,7 @@
     "secret": "{{ exchange_secret }}",
     "password": "{{ exchange_key_password }}",
     "ccxt_config": {
-        "enableRateLimit": true
+        "enableRateLimit": true,
         "rateLimit": 200
     },
     "ccxt_async_config": {

requirements-dev.txt

@@ -18,10 +18,10 @@ isort==5.9.3
 time-machine==2.4.0
 
 # Convert jupyter notebooks to markdown documents
-nbconvert==6.1.0
+nbconvert==6.2.0
 
 # mypy types
 types-cachetools==4.2.0
 types-filelock==0.1.5
-types-requests==2.25.6
+types-requests==2.25.9
 types-tabulate==0.8.2

requirements-hyperopt.txt

@@ -8,4 +8,4 @@ scikit-optimize==0.8.1
 filelock==3.0.12
 joblib==1.0.1
 psutil==5.8.0
-progressbar2==3.53.2
+progressbar2==3.53.3

requirements.txt

@@ -1,16 +1,17 @@
 numpy==1.21.2
 pandas==1.3.3
+pandas-ta==0.3.14b
 
-ccxt==1.56.30
+ccxt==1.57.3
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==3.4.8
 aiohttp==3.7.4.post0
-SQLAlchemy==1.4.23
+SQLAlchemy==1.4.25
 python-telegram-bot==13.7
 arrow==1.1.1
 cachetools==4.2.2
 requests==2.26.0
-urllib3==1.26.6
+urllib3==1.26.7
 wrapt==1.12.1
 jsonschema==3.2.0
 TA-Lib==0.4.21

scripts/rest_client.py

@@ -312,7 +312,7 @@ class FtRestClient():
         :param limit: Limit result to the last n candles.
         :return: json object
         """
-        return self._get("available_pairs", params={
+        return self._get("pair_candles", params={
             "pair": pair,
             "timeframe": timeframe,
             "limit": limit,

setup.py

@@ -54,6 +54,7 @@ setup(
         'wrapt',
         'jsonschema',
         'TA-Lib',
+        'pandas-ta',
         'technical',
         'tabulate',
         'pycoingecko',

tests/commands/test_commands.py

@@ -8,12 +8,12 @@ from zipfile import ZipFile
 import arrow
 import pytest
 
-from freqtrade.commands import (start_convert_data, start_create_userdir, start_download_data,
+from freqtrade.commands import (start_convert_data, start_convert_trades, start_create_userdir,
-                                start_hyperopt_list, start_hyperopt_show, start_install_ui,
+                                start_download_data, start_hyperopt_list, start_hyperopt_show,
-                                start_list_data, start_list_exchanges, start_list_markets,
+                                start_install_ui, start_list_data, start_list_exchanges,
-                                start_list_strategies, start_list_timeframes, start_new_strategy,
+                                start_list_markets, start_list_strategies, start_list_timeframes,
-                                start_show_trades, start_test_pairlist, start_trading,
+                                start_new_strategy, start_show_trades, start_test_pairlist,
-                                start_webserver)
+                                start_trading, start_webserver)
 from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui,
                                                 get_ui_download_url, read_ui_version)
 from freqtrade.configuration import setup_utils_configuration
@@ -208,11 +208,10 @@ def test_list_timeframes(mocker, capsys):
     assert re.search(r"^1d$", captured.out, re.MULTILINE)
 
 
-def test_list_markets(mocker, markets, capsys):
+def test_list_markets(mocker, markets_static, capsys):
 
     api_mock = MagicMock()
-    api_mock.markets = markets
+    patch_exchange(mocker, api_mock=api_mock, id='bittrex', mock_markets=markets_static)
-    patch_exchange(mocker, api_mock=api_mock, id='bittrex')
 
     # Test with no --config
     args = [
@@ -237,7 +236,7 @@ def test_list_markets(mocker, markets, capsys):
             "TKN/BTC, XLTCUSDT, XRP/BTC.\n"
             in captured.out)
 
-    patch_exchange(mocker, api_mock=api_mock, id="binance")
+    patch_exchange(mocker, api_mock=api_mock, id="binance", mock_markets=markets_static)
     # Test with --exchange
     args = [
         "list-markets",
@@ -250,7 +249,7 @@ def test_list_markets(mocker, markets, capsys):
     assert re.match("\nExchange Binance has 10 active markets:\n",
                     captured.out)
 
-    patch_exchange(mocker, api_mock=api_mock, id="bittrex")
+    patch_exchange(mocker, api_mock=api_mock, id="bittrex", mock_markets=markets_static)
     # Test with --all: all markets
     args = [
         "list-markets", "--all",
@@ -760,6 +759,22 @@ def test_download_data_trades(mocker, caplog):
     assert convert_mock.call_count == 1
 
 
+def test_start_convert_trades(mocker, caplog):
+    convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv',
+                                MagicMock(return_value=[]))
+    patch_exchange(mocker)
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
+    )
+    args = [
+        "trades-to-ohlcv",
+        "--exchange", "kraken",
+        "--pairs", "ETH/BTC", "XRP/BTC",
+    ]
+    start_convert_trades(get_args(args))
+    assert convert_mock.call_count == 1
+
+
 def test_start_list_strategies(mocker, caplog, capsys):
 
     args = [

tests/conftest.py

@@ -110,8 +110,10 @@ def patch_exchange(
     mocker.patch('freqtrade.exchange.Exchange.precisionMode', PropertyMock(return_value=2))
 
     if mock_markets:
+        if isinstance(mock_markets, bool):
+            mock_markets = get_markets()
         mocker.patch('freqtrade.exchange.Exchange.markets',
-                     PropertyMock(return_value=get_markets()))
+                     PropertyMock(return_value=mock_markets))
 
     if mock_supported_modes:
         mocker.patch(
@@ -470,6 +472,8 @@ def markets():
 
 
 def get_markets():
+    # See get_markets_static() for immutable markets and do not modify them unless absolutely
+    # necessary!
     return {
         'ETH/BTC': {
             'id': 'ethbtc',
@@ -786,11 +790,22 @@ def get_markets():
 
 
 @pytest.fixture
-def shitcoinmarkets(markets):
+def markets_static():
+    # These markets are used in some tests that would need adaptation should anything change in
+    # market list. Do not modify this list without a good reason! Do not modify market parameters
+    # of listed pairs in get_markets() without a good reason either!
+    static_markets = ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD',
+                      'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XLTCUSDT', 'XRP/BTC']
+    all_markets = get_markets()
+    return {m: all_markets[m] for m in static_markets}
+
+
+@pytest.fixture
+def shitcoinmarkets(markets_static):
     """
     Fixture with shitcoin markets - used to test filters in pairlists
     """
-    shitmarkets = deepcopy(markets)
+    shitmarkets = deepcopy(markets_static)
     shitmarkets.update({
         'HOT/BTC': {
             'id': 'HOTBTC',
@@ -1796,14 +1811,6 @@ def trades_for_order2():
              'fee': {'cost': 0.004, 'currency': 'LTC'}}]
 
 
-@pytest.fixture(scope="function")
-def trades_for_order3(trades_for_order2):
-    # Different fee currencies for each trade
-    trades_for_order = deepcopy(trades_for_order2)
-    trades_for_order[0]['fee'] = {'cost': 0.02, 'currency': 'BNB'}
-    return trades_for_order
-
-
 @pytest.fixture
 def buy_order_fee():
     return {

tests/exchange/test_exchange.py

@@ -2806,7 +2806,7 @@ def test_get_valid_pair_combination(default_conf, mocker, markets):
         (['LTC'], ['NONEXISTENT'], False, False,
          []),
     ])
-def test_get_markets(default_conf, mocker, markets,
+def test_get_markets(default_conf, mocker, markets_static,
                      base_currencies, quote_currencies, pairs_only, active_only,
                      expected_keys):
     mocker.patch.multiple('freqtrade.exchange.Exchange',
@@ -2814,7 +2814,7 @@ def test_get_markets(default_conf, mocker, markets,
                           _load_async_markets=MagicMock(),
                           validate_pairs=MagicMock(),
                           validate_timeframes=MagicMock(),
-                          markets=PropertyMock(return_value=markets))
+                          markets=PropertyMock(return_value=markets_static))
     ex = Exchange(default_conf)
     pairs = ex.get_markets(base_currencies, quote_currencies, pairs_only, active_only)
     assert sorted(pairs.keys()) == sorted(expected_keys)

tests/plugins/test_pairlist.py

@@ -131,9 +131,9 @@ def test_load_pairlist_noexist(mocker, markets, default_conf):
                                        default_conf, {}, 1)
 
 
-def test_load_pairlist_verify_multi(mocker, markets, default_conf):
+def test_load_pairlist_verify_multi(mocker, markets_static, default_conf):
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
+    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets_static))
     plm = PairListManager(freqtrade.exchange, default_conf)
     # Call different versions one after the other, should always consider what was passed in
     # and have no side-effects (therefore the same check multiple times)

tests/plugins/test_protections.py

@@ -125,7 +125,7 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog):
     # Test 5m after lock-period - this should try and relock the pair, but end-time
     # should be the previous end-time
     end_time = PairLocks.get_pair_longest_lock('*').lock_end_time + timedelta(minutes=5)
-    assert freqtrade.protections.global_stop(end_time)
+    freqtrade.protections.global_stop(end_time)
     assert not PairLocks.is_global_lock(end_time)
 
 
@@ -182,7 +182,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair
         min_ago_open=180, min_ago_close=30, profit_rate=0.9,
     ))
 
-    assert freqtrade.protections.stop_per_pair(pair)
+    freqtrade.protections.stop_per_pair(pair)
     assert freqtrade.protections.global_stop() != only_per_pair
     assert PairLocks.is_pair_locked(pair)
     assert PairLocks.is_global_lock() != only_per_pair

tests/rpc/test_rpc_telegram.py

@@ -1317,6 +1317,34 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None:
             'Reason: cancelled due to timeout.')
 
 
+def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> None:
+
+    default_conf['telegram']['notification_settings']['protection_trigger'] = 'on'
+
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+    time_machine.move_to("2021-09-01 05:00:00 +00:00")
+    lock = PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=6).datetime, 'randreason')
+    msg = {
+        'type': RPCMessageType.PROTECTION_TRIGGER,
+    }
+    msg.update(lock.to_json())
+    telegram.send_msg(msg)
+    assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. "
+            "`ETH/BTC` will be locked until `2021-09-01 05:10:00`.")
+
+    msg_mock.reset_mock()
+    # Test global protection
+
+    msg = {
+        'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL,
+    }
+    lock = PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=100).datetime, 'randreason')
+    msg.update(lock.to_json())
+    telegram.send_msg(msg)
+    assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. "
+            "*All pairs* will be locked until `2021-09-01 06:45:00`.")
+
+
 def test_send_msg_buy_fill_notification(default_conf, mocker) -> None:
 
     default_conf['telegram']['notification_settings']['buy_fill'] = 'on'

tests/test_freqtradebot.py

@@ -446,6 +446,29 @@ def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order,
     assert log_has_re(message, caplog)
 
 
+def test_handle_protections(mocker, default_conf, fee):
+    default_conf['protections'] = [
+        {"method": "CooldownPeriod", "stop_duration": 60},
+        {
+            "method": "StoplossGuard",
+            "lookback_period_candles": 24,
+            "trade_limit": 4,
+            "stop_duration_candles": 4,
+            "only_per_pair": False
+        }
+    ]
+
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    freqtrade.protections._protection_handlers[1].global_stop = MagicMock(
+        return_value=(True, arrow.utcnow().shift(hours=1).datetime, "asdf"))
+    create_mock_trades(fee)
+    freqtrade.handle_protections('ETC/BTC')
+    send_msg_mock = freqtrade.rpc.send_msg
+    assert send_msg_mock.call_count == 2
+    assert send_msg_mock.call_args_list[0][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER
+    assert send_msg_mock.call_args_list[1][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER_GLOBAL
+
+
 def test_create_trade_no_signal(default_conf, fee, mocker) -> None:
     default_conf['dry_run'] = True
 
@@ -3723,6 +3746,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe
     )
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
+    caplog.clear()
     # Amount is reduced by "fee"
     assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001)
     assert log_has(
@@ -3782,8 +3806,35 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f
     )
 
 
-def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fee, mocker):
+@pytest.mark.parametrize(
-    trades_for_order[0]['fee']['currency'] = 'ETH'
+    'fee_par,fee_reduction_amount,use_ticker_rate,expected_log', [
+        # basic, amount does not change
+        ({'cost': 0.008, 'currency': 'ETH'}, 0, False, None),
+        # no currency in fee
+        ({'cost': 0.004, 'currency': None}, 0, True, None),
+        # BNB no rate
+        ({'cost': 0.00094518, 'currency': 'BNB'}, 0, True, (
+            'Fee for Trade Trade(id=None, pair=LTC/ETH, amount=8.00000000, is_short=False, '
+            'leverage=1.0, open_rate=0.24544100, open_since=closed) [buy]: 0.00094518 BNB -'
+            ' rate: None'
+        )),
+        # from order
+        ({'cost': 0.004, 'currency': 'LTC'}, 0.004, False, (
+            'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
+            'is_short=False, leverage=1.0, open_rate=0.24544100, open_since=closed) (from'
+            ' 8.0 to 7.996).'
+        )),
+        # invalid, no currency in from fee dict
+        ({'cost': 0.008, 'currency': None}, 0, True, None),
+    ])
+def test_get_real_amount(
+    default_conf, trades_for_order, buy_order_fee, fee, mocker, caplog,
+    fee_par, fee_reduction_amount, use_ticker_rate, expected_log
+):
+
+    buy_order = deepcopy(buy_order_fee)
+    buy_order['fee'] = fee_par
+    trades_for_order[0]['fee'] = fee_par
 
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
     amount = sum(x['amount'] for x in trades_for_order)
@@ -3798,19 +3849,38 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fe
     )
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
-    # Amount does not change
+    if not use_ticker_rate:
-    assert freqtrade.get_real_amount(trade, buy_order_fee) == amount
+        mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', side_effect=ExchangeError)
+
+    caplog.clear()
+    assert freqtrade.get_real_amount(trade, buy_order) == amount - fee_reduction_amount
+
+    if expected_log:
+        assert log_has(expected_log, caplog)
 
 
-def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_order_fee,
+@pytest.mark.parametrize(
-                                            fee, mocker):
+    'fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount', [
+        # basic, amount is reduced by fee
+        (None, None, 0.001, 0.001, 7.992),
+        # different fee currency on both trades, fee is average of both trade's fee
+        (0.02, 'BNB', 0.0005, 0.001518575, 7.996),
+    ])
+def test_get_real_amount_multi(
+    default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker, markets,
+    fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount,
+):
 
-    limit_buy_order = deepcopy(buy_order_fee)
+    trades_for_order = deepcopy(trades_for_order2)
-    limit_buy_order['fee'] = {'cost': 0.004, 'currency': None}
+    if fee_cost:
-    trades_for_order[0]['fee']['currency'] = None
+        trades_for_order[0]['fee']['cost'] = fee_cost
+    if fee_currency:
+        trades_for_order[0]['fee']['currency'] = fee_currency
 
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
-    amount = sum(x['amount'] for x in trades_for_order)
+    amount = float(sum(x['amount'] for x in trades_for_order))
+    default_conf['stake_currency'] = "ETH"
+
     trade = Trade(
         pair='LTC/ETH',
         amount=amount,
@@ -3820,78 +3890,7 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
-    # Amount does not change
-    assert freqtrade.get_real_amount(trade, limit_buy_order) == amount
-
-
-def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, fee, mocker):
-    trades_for_order[0]['fee']['currency'] = 'BNB'
-    trades_for_order[0]['fee']['cost'] = 0.00094518
-
-    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
-    amount = sum(x['amount'] for x in trades_for_order)
-    trade = Trade(
-        pair='LTC/ETH',
-        amount=amount,
-        exchange='binance',
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        open_rate=0.245441,
-        open_order_id="123456"
-    )
-    freqtrade = get_patched_freqtradebot(mocker, default_conf)
-
-    # Amount does not change
-    assert freqtrade.get_real_amount(trade, buy_order_fee) == amount
-
-
-def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker):
-    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order2)
-    amount = float(sum(x['amount'] for x in trades_for_order2))
-    trade = Trade(
-        pair='LTC/ETH',
-        amount=amount,
-        exchange='binance',
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        open_rate=0.245441,
-        open_order_id="123456"
-    )
-    freqtrade = get_patched_freqtradebot(mocker, default_conf)
-
-    # Amount is reduced by "fee"
-    assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001)
-    assert log_has(
-        'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, is_short=False,'
-        ' leverage=1.0, open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).',
-        caplog
-    )
-
-    assert trade.fee_open == 0.001
-    assert trade.fee_close == 0.001
-    assert trade.fee_open_cost is not None
-    assert trade.fee_open_currency is not None
-    assert trade.fee_close_cost is None
-    assert trade.fee_close_currency is None
-
-
-def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, caplog, fee,
-                                mocker, markets):
-    # Different fee currency on both trades
-    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order3)
-    amount = float(sum(x['amount'] for x in trades_for_order3))
-    default_conf['stake_currency'] = 'ETH'
-    trade = Trade(
-        pair='LTC/ETH',
-        amount=amount,
-        exchange='binance',
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        open_rate=0.245441,
-        open_order_id="123456"
-    )
     # Fake markets entry to enable fee parsing
     markets['BNB/ETH'] = markets['ETH/BTC']
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
@@ -3900,50 +3899,25 @@ def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee,
                  return_value={'ask': 0.19, 'last': 0.2})
 
     # Amount is reduced by "fee"
-    assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.0005)
+    expected_amount = amount - (amount * fee_reduction_amount)
+    assert freqtrade.get_real_amount(trade, buy_order_fee) == expected_amount
     assert log_has(
-        'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, is_short=False,'
+        (
-        ' leverage=1.0, open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).',
+            'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
+            'is_short=False, leverage=1.0, open_rate=0.24544100, open_since=closed) '
+            f'(from 8.0 to {expected_log_amount}).'
+        ),
         caplog
     )
-    # Overall fee is average of both trade's fee
+
-    assert trade.fee_open == 0.001518575
+    assert trade.fee_open == expected_fee
+    assert trade.fee_close == expected_fee
     assert trade.fee_open_cost is not None
     assert trade.fee_open_currency is not None
     assert trade.fee_close_cost is None
     assert trade.fee_close_currency is None
 
 
-def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, fee,
-                                   caplog, mocker):
-    limit_buy_order = deepcopy(buy_order_fee)
-    limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'}
-
-    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order',
-                 return_value=[trades_for_order])
-    amount = float(sum(x['amount'] for x in trades_for_order))
-    trade = Trade(
-        pair='LTC/ETH',
-        amount=amount,
-        exchange='binance',
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        open_rate=0.245441,
-        open_order_id="123456"
-    )
-    freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    # Ticker rate cannot be found for this to work.
-    mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', side_effect=ExchangeError)
-
-    # Amount is reduced by "fee"
-    assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004
-    assert log_has(
-        'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, is_short=False,'
-        ' leverage=1.0, open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).',
-        caplog
-    )
-
-
 def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, fee, mocker):
     limit_buy_order = deepcopy(buy_order_fee)
     limit_buy_order['fee'] = {'cost': 0.004}
@@ -4011,27 +3985,6 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b
                    abs_tol=MATH_CLOSE_PREC,)
 
 
-def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, fee, mocker):
-    # Remove "Currency" from fee dict
-    trades_for_order[0]['fee'] = {'cost': 0.008}
-
-    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
-    amount = sum(x['amount'] for x in trades_for_order)
-    trade = Trade(
-        pair='LTC/ETH',
-        amount=amount,
-        exchange='binance',
-        open_rate=0.245441,
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-
-        open_order_id="123456"
-    )
-    freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    # Amount does not change
-    assert freqtrade.get_real_amount(trade, buy_order_fee) == amount
-
-
 def test_get_real_amount_open_trade(default_conf, fee, mocker):
     amount = 12345
     trade = Trade(
@@ -4082,10 +4035,14 @@ def test_apply_fee_conditional(default_conf, fee, caplog, mocker,
     assert walletmock.call_count == 1
 
 
+@pytest.mark.parametrize("delta, is_high_delta", [
+    (0.1, False),
+    (100, True),
+])
 def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open, limit_buy_order,
-                                    is_short, fee, mocker, order_book_l2):
+                                    is_short, fee, mocker, order_book_l2, delta, is_high_delta):
     default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True
-    default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1
+    default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = delta
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2)
@@ -4103,6 +4060,9 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open,
     freqtrade.enter_positions()
 
     trade = Trade.query.first()
+    if is_high_delta:
+        assert trade is None
+    else:
         assert trade is not None
         assert trade.stake_amount == 0.001
         assert trade.is_open
@@ -4118,29 +4078,6 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open,
         assert whitelist == default_conf['exchange']['pair_whitelist']
 
 
-def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_order,
-                                               fee, mocker, order_book_l2):
-    default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True
-    # delta is 100 which is impossible to reach. hence check_depth_of_market will return false
-    default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
-    mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2)
-    mocker.patch.multiple(
-        'freqtrade.exchange.Exchange',
-        fetch_ticker=ticker,
-        create_order=MagicMock(return_value={'id': limit_buy_order['id']}),
-        get_fee=fee,
-    )
-    # Save state of current whitelist
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
-    freqtrade.enter_positions()
-
-    trade = Trade.query.first()
-    assert trade is None
-
-
 @pytest.mark.parametrize('exception_thrown,ask,last,order_book_top,order_book', [
     (False, 0.045, 0.046, 2, None),
     (True,  0.042, 0.046, 1, {'bids': [[]], 'asks': [[]]})
@@ -4341,16 +4278,16 @@ def test_check_for_open_trades(mocker, default_conf, fee, is_short):
 
 @pytest.mark.parametrize("is_short", [False, True])
 @pytest.mark.usefixtures("init_persistence")
-def test_update_open_orders(mocker, default_conf, fee, caplog, is_short):
+def test_startup_update_open_orders(mocker, default_conf, fee, caplog, is_short):
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
     create_mock_trades(fee, is_short=is_short)
 
-    freqtrade.update_open_orders()
+    freqtrade.startup_update_open_orders()
     assert not log_has_re(r"Error updating Order .*", caplog)
     caplog.clear()
 
     freqtrade.config['dry_run'] = False
-    freqtrade.update_open_orders()
+    freqtrade.startup_update_open_orders()
 
     assert log_has_re(r"Error updating Order .*", caplog)
     caplog.clear()
@@ -4361,7 +4298,7 @@ def test_update_open_orders(mocker, default_conf, fee, caplog, is_short):
         'status': 'closed',
     })
     mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=matching_buy_order)
-    freqtrade.update_open_orders()
+    freqtrade.startup_update_open_orders()
     # Only stoploss and sell orders are kept open
     assert len(Order.get_open_orders()) == 2