Merge pull request #5491 from freqtrade/new_release

New release 2021.8

README.md

@@ -26,8 +26,8 @@ hesitate to read the source code and understand the mechanism of this bot.
 
 Please read the [exchange specific notes](docs/exchanges.md) to learn about eventual, special configurations needed for each exchange.
 
+- [X] [Binance](https://www.binance.com/) ([*Note for binance users](docs/exchanges.md#binance-blacklist))
 - [X] [Bittrex](https://bittrex.com/)
-- [X] [Binance](https://www.binance.com/) ([*Note for binance users](docs/exchanges.md#blacklists))
 - [X] [Kraken](https://kraken.com/)
 - [X] [FTX](https://ftx.com)
 - [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_
@@ -37,7 +37,7 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even
 Exchanges confirmed working by the community:
 
 - [X] [Bitvavo](https://bitvavo.com/)
-- [X] [Kukoin](https://www.kucoin.com/)
+- [X] [Kucoin](https://www.kucoin.com/)
 
 ## Documentation
 

build_helpers/publish_docker_arm64.sh

@@ -37,12 +37,12 @@ fi
 # Tag image for upload and next build step
 docker tag freqtrade:$TAG_ARM ${CACHE_IMAGE}:$TAG_ARM
 
-docker build --cache-from freqtrade:${TAG_ARM} --build-arg sourceimage=${TAG_ARM} -t freqtrade:${TAG_PLOT_ARM} -f docker/Dockerfile.plot .
+docker build --cache-from freqtrade:${TAG_ARM} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_PLOT_ARM} -f docker/Dockerfile.plot .
 
 docker tag freqtrade:$TAG_PLOT_ARM ${CACHE_IMAGE}:$TAG_PLOT_ARM
 
 # Run backtest
-docker run --rm -v $(pwd)/config_examples/config_bittrex.example.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG_ARM} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy
+docker run --rm -v $(pwd)/config_examples/config_bittrex.example.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG_ARM} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV2
 
 if [ $? -ne 0 ]; then
     echo "failed running backtest"
@@ -63,18 +63,16 @@ echo "create manifests"
 docker manifest create --amend ${IMAGE_NAME}:${TAG} ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} ${CACHE_IMAGE}:${TAG}
 docker manifest push -p ${IMAGE_NAME}:${TAG}
 
-docker manifest create --amend ${IMAGE_NAME}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT_ARM} ${CACHE_IMAGE}:${TAG_PLOT}
+docker manifest create ${IMAGE_NAME}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT_ARM} ${CACHE_IMAGE}:${TAG_PLOT}
 docker manifest push -p ${IMAGE_NAME}:${TAG_PLOT}
 
-Tag as latest for develop builds
+# Tag as latest for develop builds
 if [ "${TAG}" = "develop" ]; then
-    docker tag ${IMAGE_NAME}:develop ${IMAGE_NAME}:latest
+    docker manifest create ${IMAGE_NAME}:latest ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} ${CACHE_IMAGE}:${TAG}
-    docker push ${IMAGE_NAME}:latest
+    docker manifest push -p ${IMAGE_NAME}:latest
 fi
 
 docker images
 
-if [ $? -ne 0 ]; then
+# Cleanup old images from arm64 node.
-    echo "failed building image"
+docker image prune -a --force --filter "until=24h"
-    return 1
-fi

build_helpers/publish_docker_multi.sh

@@ -48,12 +48,12 @@ fi
 # Tag image for upload and next build step
 docker tag freqtrade:$TAG ${CACHE_IMAGE}:$TAG
 
-docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot .
+docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot .
 
 docker tag freqtrade:$TAG_PLOT ${CACHE_IMAGE}:$TAG_PLOT
 
 # Run backtest
-docker run --rm -v $(pwd)/config_examples/config_bittrex.example.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy
+docker run --rm -v $(pwd)/config_examples/config_bittrex.example.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV2
 
 if [ $? -ne 0 ]; then
     echo "failed running backtest"

config_examples/config_full.example.json

@@ -78,33 +78,6 @@
             "refresh_period": 1440
         }
     ],
-    "protections": [
-        {
-            "method": "StoplossGuard",
-            "lookback_period_candles": 60,
-            "trade_limit": 4,
-            "stop_duration_candles": 60,
-            "only_per_pair": false
-        },
-        {
-            "method": "CooldownPeriod",
-            "stop_duration_candles": 20
-        },
-        {
-            "method": "MaxDrawdown",
-            "lookback_period_candles": 200,
-            "trade_limit": 20,
-            "stop_duration_candles": 10,
-            "max_allowed_drawdown": 0.2
-        },
-        {
-            "method": "LowProfitPairs",
-            "lookback_period_candles": 360,
-            "trade_limit": 1,
-            "stop_duration_candles": 2,
-            "required_profit": 0.02
-        }
-    ],
     "exchange": {
         "name": "binance",
         "sandbox": false,
@@ -201,7 +174,7 @@
         "heartbeat_interval": 60
     },
     "disable_dataframe_checks": false,
-    "strategy": "DefaultStrategy",
+    "strategy": "SampleStrategy",
     "strategy_path": "user_data/strategies/",
     "dataformat_ohlcv": "json",
     "dataformat_trades": "jsongz"

docker/Dockerfile.plot

@@ -1,5 +1,6 @@
-ARG sourceimage=develop
+ARG sourceimage=freqtradeorg/freqtrade
-FROM freqtradeorg/freqtrade:${sourceimage}
+ARG sourcetag=develop
+FROM ${sourceimage}:${sourcetag}
 
 # Install dependencies
 COPY requirements-plot.txt /freqtrade/

docs/backtesting.md

@@ -62,7 +62,7 @@ optional arguments:
                         this together with `--export trades`, the strategy-
                         name is injected into the filename (so `backtest-
                         data.json` becomes `backtest-data-
-                        DefaultStrategy.json`
+                        SampleStrategy.json`
   --export {none,trades}
                         Export backtest results (default: trades).
   --export-filename PATH

docs/bot-basics.md

@@ -35,12 +35,13 @@ By default, loop runs every few seconds (`internals.process_throttle_secs`) and
   * Calls `check_buy_timeout()` strategy callback for open buy orders.
   * Calls `check_sell_timeout()` strategy callback for open sell orders.
 * Verifies existing positions and eventually places sell orders.
-  * Considers stoploss, ROI and sell-signal.
+  * Considers stoploss, ROI and sell-signal, `custom_sell()` and `custom_stoploss()`.
-  * Determine sell-price based on `ask_strategy` configuration setting.
+  * Determine sell-price based on `ask_strategy` configuration setting or by using the `custom_exit_price()` callback.
   * Before a sell order is placed, `confirm_trade_exit()` strategy callback is called.
 * Check if trade-slots are still available (if `max_open_trades` is reached).
 * Verifies buy signal trying to enter new positions.
-  * Determine buy-price based on `bid_strategy` configuration setting.
+  * Determine buy-price based on `bid_strategy` configuration setting, or by using the `custom_entry_price()` callback.
+  * Determine stake size by calling the `custom_stake_amount()` callback.
   * Before a buy order is placed, `confirm_trade_entry()` strategy callback is called.
 
 This loop will be repeated again and again until the bot is stopped.
@@ -52,9 +53,10 @@ This loop will be repeated again and again until the bot is stopped.
 * Load historic data for configured pairlist.
 * Calls `bot_loop_start()` once.
 * Calculate indicators (calls `populate_indicators()` once per pair).
-* Calculate buy / sell signals (calls `populate_buy_trend()` and `populate_sell_trend()` once per pair)
+* Calculate buy / sell signals (calls `populate_buy_trend()` and `populate_sell_trend()` once per pair).
-* Confirm trade buy / sell (calls `confirm_trade_entry()` and `confirm_trade_exit()` if implemented in the strategy)
 * Loops per candle simulating entry and exit points.
+  * Confirm trade buy / sell (calls `confirm_trade_entry()` and `confirm_trade_exit()` if implemented in the strategy).
+  * Call `custom_stoploss()` and `custom_sell()` to find custom exit points.
 * Generate backtest report output
 
 !!! Note

docs/configuration.md

@@ -11,6 +11,37 @@ Per default, the bot loads the configuration from the `config.json` file, locate
 
 You can specify a different configuration file used by the bot with the `-c/--config` command-line option.
 
+If you used the [Quick start](installation.md/#quick-start) method for installing 
+the bot, the installation script should have already created the default configuration file (`config.json`) for you.
+
+If the default configuration file is not created we recommend to use `freqtrade new-config --config config.json` to generate a basic configuration file.
+
+The Freqtrade configuration file is to be written in JSON format.
+
+Additionally to the standard JSON syntax, you may use one-line `// ...` and multi-line `/* ... */` comments in your configuration files and trailing commas in the lists of parameters.
+
+Do not worry if you are not familiar with JSON format -- simply open the configuration file with an editor of your choice, make some changes to the parameters you need, save your changes and, finally, restart the bot or, if it was previously stopped, run it again with the changes you made to the configuration. The bot validates the syntax of the configuration file at startup and will warn you if you made any errors editing it, pointing out problematic lines.
+
+### Environment variables
+
+Set options in the Freqtrade configuration via environment variables.
+This takes priority over the corresponding value in configuration or strategy.
+
+Environment variables must be prefixed with  `FREQTRADE__` to be loaded to the freqtrade configuration.
+
+`__` serves as level separator, so the format used should correspond to `FREQTRADE__{section}__{key}`.
+As such - an environment variable defined as  `export FREQTRADE__STAKE_AMOUNT=200` would result in `{stake_amount: 200}`.
+
+A more complex example might be `export FREQTRADE__EXCHANGE__KEY=<yourExchangeKey>` to keep your exchange key secret. This will move the value to the `exchange.key` section of the configuration.
+Using this scheme, all configuration settings will also be available as environment variables.
+
+Please note that Environment variables will overwrite corresponding settings in your configuration, but command line Arguments will always win.
+
+!!! Note
+    Environment variables detected are logged at startup - so if you can't find why a value is not what you think it should be based on the configuration, make sure it's not loaded from an environment variable.
+
+### Multiple configuration files
+
 Multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream.
 
 !!! Tip "Use multiple configuration files to keep secrets secret"
@@ -22,17 +53,6 @@ Multiple configuration files can be specified and used by the bot or the bot can
     The 2nd file should only specify what you intend to override.
     If a key is in more than one of the configurations, then the "last specified configuration" wins (in the above example, `config-private.json`).
 
-If you used the [Quick start](installation.md/#quick-start) method for installing 
-the bot, the installation script should have already created the default configuration file (`config.json`) for you.
-
-If the default configuration file is not created we recommend you to use `freqtrade new-config --config config.json` to generate a basic configuration file.
-
-The Freqtrade configuration file is to be written in JSON format.
-
-Additionally to the standard JSON syntax, you may use one-line `// ...` and multi-line `/* ... */` comments in your configuration files and trailing commas in the lists of parameters.
-
-Do not worry if you are not familiar with JSON format -- simply open the configuration file with an editor of your choice, make some changes to the parameters you need, save your changes and, finally, restart the bot or, if it was previously stopped, run it again with the changes you made to the configuration. The bot validates the syntax of the configuration file at startup and will warn you if you made any errors editing it, pointing out problematic lines.
-
 ## Configuration parameters
 
 The table below will list all configuration parameters available.
@@ -41,6 +61,7 @@ Freqtrade can also load many options via command line (CLI) arguments (check out
 The prevalence for all Options is as follows:
 
 - CLI arguments override any other option
+- [Environment Variables](#environment-variables)
 - Configuration files are used in sequence (the last file wins) and override Strategy configurations.
 - Strategy configurations are only used if they are not set via configuration or command-line arguments. These options are marked with [Strategy Override](#parameters-in-the-strategy) in the below table.
 
@@ -84,11 +105,12 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `ask_strategy.order_book_top` | Bot will use the top N rate in Order Book "price_side" to sell. I.e. a value of 2 will allow the bot to pick the 2nd ask rate in [Order Book Asks](#sell-price-with-orderbook-enabled)<br>*Defaults to `1`.* <br> **Datatype:** Positive Integer
 | `use_sell_signal` | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `true`.* <br> **Datatype:** Boolean
 | `sell_profit_only` | Wait until the bot reaches `sell_profit_offset` before taking a sell decision. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.* <br> **Datatype:** Boolean
-| `sell_profit_offset` | Sell-signal is only active above this value. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `0.0`.* <br> **Datatype:** Float (as ratio)
+| `sell_profit_offset` | Sell-signal is only active above this value. Only active in combination with `sell_profit_only=True`. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `0.0`.* <br> **Datatype:** Float (as ratio)
 | `ignore_roi_if_buy_signal` | Do not sell if the buy signal is still active. This setting takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.* <br> **Datatype:** Boolean
 | `ignore_buying_expired_candle_after` | Specifies the number of seconds until a buy signal is no longer used. <br> **Datatype:** Integer
 | `order_types` | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Dict
 | `order_time_in_force` | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Dict
+| `custom_price_max_distance_ratio` | Configure maximum distance ratio between current and custom entry or exit price. <br>*Defaults to `0.02` 2%).*<br> **Datatype:** Positive float
 | `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). <br> **Datatype:** String
 | `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.<br> **Datatype:** Boolean
 | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
@@ -526,9 +548,10 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo
 
 ## Switch to production mode
 
-In production mode, the bot will engage your money. Be careful, since a wrong
+In production mode, the bot will engage your money. Be careful, since a wrong strategy can lose all your money.
-strategy can lose all your money. Be aware of what you are doing when
+Be aware of what you are doing when you run it in production mode.
-you run it in production mode.
+
+When switching to Production mode, please make sure to use a different / fresh database to avoid dry-run trades messing with your exchange money and eventually tainting your statistics.
 
 ### Setup your exchange account
 

docs/developer.md

@@ -240,11 +240,18 @@ The `IProtection` parent class provides a helper method for this in `calculate_l
 !!! Note
     This section is a Work in Progress and is not a complete guide on how to test a new exchange with Freqtrade.
 
+!!! Note
+    Make sure to use an up-to-date version of CCXT before running any of the below tests.
+    You can get the latest version of ccxt by running `pip install -U ccxt` with activated virtual environment.
+    Native docker is not supported for these tests, however the available dev-container will support all required actions and eventually necessary changes.
+
 Most exchanges supported by CCXT should work out of the box.
 
 To quickly test the public endpoints of an exchange, add a configuration for your exchange to `test_ccxt_compat.py` and run these tests with `pytest --longrun tests/exchange/test_ccxt_compat.py`.
 Completing these tests successfully a good basis point (it's a requirement, actually), however these won't guarantee correct exchange functioning, as this only tests public endpoints, but no private endpoint (like generate order or similar).
 
+Also try to use `freqtrade download-data` for an extended timerange and verify that the data downloaded correctly (no holes, the specified timerange was actually downloaded).
+
 ### Stoploss On Exchange
 
 Check if the new exchange supports Stoploss on Exchange orders through their API.

docs/exchanges.md

@@ -105,7 +105,7 @@ To use subaccounts with FTX, you need to edit the configuration and add the foll
 
 ## Kucoin
 
-Kucoin requries a passphrase for each api key, you will therefore need to add this key into the configuration so your exchange section looks as follows:
+Kucoin requires a passphrase for each api key, you will therefore need to add this key into the configuration so your exchange section looks as follows:
 
 ```json
 "exchange": {

docs/hyperopt.md

@@ -48,7 +48,7 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
                           [--hyperopt-path PATH] [--eps] [--dmmp]
                           [--enable-protections]
                           [--dry-run-wallet DRY_RUN_WALLET] [-e INT]
-                          [--spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]]
+                          [--spaces {all,buy,sell,roi,stoploss,trailing,protection,default} [{all,buy,sell,roi,stoploss,trailing,protection,default} ...]]
                           [--print-all] [--no-color] [--print-json] [-j JOBS]
                           [--random-state INT] [--min-trades INT]
                           [--hyperopt-loss NAME] [--disable-param-export]
@@ -92,7 +92,7 @@ optional arguments:
                         Starting balance, used for backtesting / hyperopt and
                         dry-runs.
   -e INT, --epochs INT  Specify number of epochs (default: 100).
-  --spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]
+  --spaces {all,buy,sell,roi,stoploss,trailing,protection,default} [{all,buy,sell,roi,stoploss,trailing,protection,default} ...]
                         Specify which parameters to hyperopt. Space-separated
                         list.
   --print-all           Print all results, not only the best ones.
@@ -253,7 +253,7 @@ We continue to define hyperoptable parameters:
 class MyAwesomeStrategy(IStrategy):
     buy_adx = DecimalParameter(20, 40, decimals=1, default=30.1, space="buy")
     buy_rsi = IntParameter(20, 40, default=30, space="buy")
-    buy_adx_enabled = CategoricalParameter([True, False], default=True, space="buy")
+    buy_adx_enabled = BooleanParameter(default=True, space="buy")
     buy_rsi_enabled = CategoricalParameter([True, False], default=False, space="buy")
     buy_trigger = CategoricalParameter(["bb_lower", "macd_cross_signal"], default="bb_lower", space="buy")
 ```
@@ -316,6 +316,7 @@ There are four parameter types each suited for different purposes.
 * `DecimalParameter` - defines a floating point parameter with a limited number of decimals (default 3). Should be preferred instead of `RealParameter` in most cases.
 * `RealParameter` - defines a floating point parameter with upper and lower boundaries and no precision limit. Rarely used as it creates a space with a near infinite number of possibilities.
 * `CategoricalParameter` - defines a parameter with a predetermined number of choices.
+* `BooleanParameter` - Shorthand for `CategoricalParameter([True, False])` - great for "enable" parameters.
 
 !!! Tip "Disabling parameter optimization"
     Each parameter takes two boolean parameters:
@@ -326,7 +327,7 @@ There are four parameter types each suited for different purposes.
 !!! Warning
     Hyperoptable parameters cannot be used in `populate_indicators` - as hyperopt does not recalculate indicators for each epoch, so the starting value would be used in this case.
 
-### Optimizing an indicator parameter
+## Optimizing an indicator parameter
 
 Assuming you have a simple strategy in mind - a EMA cross strategy (2 Moving averages crossing) - and you'd like to find the ideal parameters for this strategy.
 
@@ -336,8 +337,8 @@ from functools import reduce
 
 import talib.abstract as ta
 
-from freqtrade.strategy import IStrategy
+from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter, 
-from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter
+                                IStrategy, IntParameter)
 import freqtrade.vendor.qtpylib.indicators as qtpylib
 
 class MyAwesomeStrategy(IStrategy):
@@ -413,6 +414,98 @@ While this strategy is most likely too simple to provide consistent profit, it s
     While this may slow down the hyperopt startup speed, the overall performance will increase as the Hyperopt execution itself may pick the same value for multiple epochs (changing other values).
     You should however try to use space ranges as small as possible. Every new column will require more memory, and every possibility hyperopt can try will increase the search space.
 
+## Optimizing protections
+
+Freqtrade can also optimize protections. How you optimize protections is up to you, and the following should be considered as example only.
+
+The strategy will simply need to define the "protections" entry as property returning a list of protection configurations.
+
+``` python
+from pandas import DataFrame
+from functools import reduce
+
+import talib.abstract as ta
+
+from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter, 
+                                IStrategy, IntParameter)
+import freqtrade.vendor.qtpylib.indicators as qtpylib
+
+class MyAwesomeStrategy(IStrategy):
+    stoploss = -0.05
+    timeframe = '15m'
+    # Define the parameter spaces
+    cooldown_lookback = IntParameter(2, 48, default=5, space="protection", optimize=True)
+    stop_duration = IntParameter(12, 200, default=5, space="protection", optimize=True)
+    use_stop_protection = BooleanParameter(default=True, space="protection", optimize=True)
+
+
+    @property
+    def protections(self):
+        prot = []
+
+        prot.append({
+            "method": "CooldownPeriod",
+            "stop_duration_candles": self.cooldown_lookback.value
+        })
+        if self.use_stop_protection.value:
+            prot.append({
+                "method": "StoplossGuard",
+                "lookback_period_candles": 24 * 3,
+                "trade_limit": 4,
+                "stop_duration_candles": self.stop_duration.value,
+                "only_per_pair": False
+            })
+
+        return protection
+
+    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        # ...
+        
+```
+
+You can then run hyperopt as follows:
+`freqtrade hyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy MyAwesomeStrategy --spaces protection`
+
+!!! Note
+    The protection space is not part of the default space, and is only available with the Parameters Hyperopt interface, not with the legacy hyperopt interface (which required separate hyperopt files).
+    Freqtrade will also automatically change the "--enable-protections" flag if the protection space is selected.
+
+!!! Warning
+    If protections are defined as property, entries from the configuration will be ignored.
+    It is therefore recommended to not define protections in the configuration.
+
+### Migrating from previous property setups
+
+A migration from a previous setup is pretty simple, and can be accomplished by converting the protections entry to a property.
+In simple terms, the following configuration will be converted to the below.
+
+``` python
+class MyAwesomeStrategy(IStrategy):
+    protections = [
+        {
+            "method": "CooldownPeriod",
+            "stop_duration_candles": 4
+        }
+    ]
+```
+
+Result
+
+``` python
+class MyAwesomeStrategy(IStrategy):
+    
+    @property
+    def protections(self):
+        return [
+            {
+                "method": "CooldownPeriod",
+                "stop_duration_candles": 4
+            }
+        ]
+```
+
+You will then obviously also change potential interesting entries to parameters to allow hyper-optimization.
+
 ## Loss-functions
 
 Each hyperparameter tuning requires a target. This is usually defined as a loss function (sometimes also called objective function), which should decrease for more desirable results, and increase for bad results.
@@ -483,7 +576,8 @@ Legal values are:
 * `roi`: just optimize the minimal profit table for your strategy
 * `stoploss`: search for the best stoploss value
 * `trailing`: search for the best trailing stop values
-* `default`: `all` except `trailing`
+* `protection`: search for the best protection parameters (read the [protections section](#optimizing-protections) on how to properly define these)
+* `default`: `all` except `trailing` and `protection`
 * space-separated list of any of the above values for example `--spaces roi stoploss`
 
 The default Hyperopt Search Space, used when no `--space` command line option is specified, does not include the `trailing` hyperspace. We recommend you to run optimization for the `trailing` hyperspace separately, when the best parameters for other hyperspaces were found, validated and pasted into your custom strategy.

docs/includes/pairlists.md

@@ -58,7 +58,7 @@ This option must be configured along with `exchange.skip_pair_validation` in the
 
 When used in the chain of Pairlist Handlers in a non-leading position (after StaticPairList and other Pairlist Filters), `VolumePairList` considers outputs of previous Pairlist Handlers, adding its sorting/selection of the pairs by the trading volume.
 
-When used on the leading position of the chain of Pairlist Handlers, it does not consider `pair_whitelist` configuration setting, but selects the top assets from all available markets (with matching stake-currency) on the exchange.
+When used in the leading position of the chain of Pairlist Handlers, the `pair_whitelist` configuration setting is ignored. Instead, `VolumePairList` selects the top assets from all available markets with matching stake-currency on the exchange.
 
 The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes).
 The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists.
@@ -74,11 +74,14 @@ Filtering instances (not the first position in the list) will not apply any cach
         "method": "VolumePairList",
         "number_assets": 20,
         "sort_key": "quoteVolume",
+        "min_value": 0,
         "refresh_period": 1800
     }
 ],
 ```
 
+You can define a minimum volume with `min_value` - which will filter out pairs with a volume lower than the specified value in the specified timerange.
+
 `VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a  more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
 
 For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days:
@@ -89,6 +92,7 @@ For convenience `lookback_days` can be specified, which will imply that 1d candl
         "method": "VolumePairList",
         "number_assets": 20,
         "sort_key": "quoteVolume",
+        "min_value": 0,
         "refresh_period": 86400,
         "lookback_days": 7
     }
@@ -109,6 +113,7 @@ More sophisticated approach can be used, by using `lookback_timeframe` for candl
         "method": "VolumePairList",
         "number_assets": 20,
         "sort_key": "quoteVolume",
+        "min_value": 0,
         "refresh_period": 3600,
         "lookback_timeframe": "1h",
         "lookback_period": 72
@@ -221,10 +226,10 @@ If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027, the ratio
 
 #### RangeStabilityFilter
 
-Removes pairs where the difference between lowest low and highest high over `lookback_days` days is below `min_rate_of_change`. Since this is a filter that requires additional data, the results are cached for `refresh_period`.
+Removes pairs where the difference between lowest low and highest high over `lookback_days` days is below `min_rate_of_change` or above `max_rate_of_change`. Since this is a filter that requires additional data, the results are cached for `refresh_period`.
 
 In the below example:
-If the trading range over the last 10 days is <1%, remove the pair from the whitelist.
+If the trading range over the last 10 days is <1% or >99%, remove the pair from the whitelist.
 
 ```json
 "pairlists": [
@@ -232,6 +237,7 @@ If the trading range over the last 10 days is <1%, remove the pair from the whit
         "method": "RangeStabilityFilter",
         "lookback_days": 10,
         "min_rate_of_change": 0.01,
+        "max_rate_of_change": 0.99,
         "refresh_period": 1440
     }
 ]
@@ -239,6 +245,7 @@ If the trading range over the last 10 days is <1%, remove the pair from the whit
 
 !!! Tip
     This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit.
+    Additionally, it can also be used to automatically remove pairs with extreme high/low variance over a given amount of time.
 
 #### VolatilityFilter
 

docs/includes/protections.md

@@ -15,6 +15,10 @@ All protection end times are rounded up to the next candle to avoid sudden, unex
 !!! Note "Backtesting"
     Protections are supported by backtesting and hyperopt, but must be explicitly enabled by using the `--enable-protections` flag.
 
+!!! Warning "Setting protections from the configuration"
+    Setting protections from the configuration via `"protections": [],` key should be considered deprecated and will be removed in a future version.
+    It is also no longer guaranteed that your protections apply to the strategy in cases where the strategy defines [protections as property](hyperopt.md#optimizing-protections).
+
 ### Available Protections
 
 * [`StoplossGuard`](#stoploss-guard) Stop trading if a certain amount of stoploss occurred within a certain time window.
@@ -47,15 +51,17 @@ This applies across all pairs, unless `only_per_pair` is set to true, which will
 The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles.
 
 ``` python
-protections = [
+@property
-    {
+def protections(self):
-        "method": "StoplossGuard",
+    return [
-        "lookback_period_candles": 24,
+        {
-        "trade_limit": 4,
+            "method": "StoplossGuard",
-        "stop_duration_candles": 4,
+            "lookback_period_candles": 24,
-        "only_per_pair": False
+            "trade_limit": 4,
-    }
+            "stop_duration_candles": 4,
-]
+            "only_per_pair": False
+        }
+    ]
 ```
 
 !!! Note
@@ -69,15 +75,17 @@ protections = [
 The below sample stops trading for 12 candles if max-drawdown is > 20% considering all pairs - with a minimum of `trade_limit` trades - within the last 48 candles. If desired, `lookback_period` and/or `stop_duration` can be used.
 
 ``` python
-protections = [
+@property
-      {
+def protections(self):
-        "method": "MaxDrawdown",
+    return  [
-        "lookback_period_candles": 48,
+        {
-        "trade_limit": 20,
+            "method": "MaxDrawdown",
-        "stop_duration_candles": 12,
+            "lookback_period_candles": 48,
-        "max_allowed_drawdown": 0.2
+            "trade_limit": 20,
-      },
+            "stop_duration_candles": 12,
-]
+            "max_allowed_drawdown": 0.2
+        },
+    ]
 ```
 
 #### Low Profit Pairs
@@ -88,15 +96,17 @@ If that ratio is below `required_profit`, that pair will be locked for `stop_dur
 The below example will stop trading a pair for 60 minutes if the pair does not have a required profit of 2% (and a minimum of 2 trades) within the last 6 candles.
 
 ``` python
-protections = [
+@property
-    {
+def protections(self):
-        "method": "LowProfitPairs",
+    return [
-        "lookback_period_candles": 6,
+        {
-        "trade_limit": 2,
+            "method": "LowProfitPairs",
-        "stop_duration": 60,
+            "lookback_period_candles": 6,
-        "required_profit": 0.02
+            "trade_limit": 2,
-    }
+            "stop_duration": 60,
-]
+            "required_profit": 0.02
+        }
+    ]
 ```
 
 #### Cooldown Period
@@ -106,12 +116,14 @@ protections = [
 The below example will stop trading a pair for 2 candles after closing a trade, allowing this pair to "cool down".
 
 ``` python
-protections = [
+@property
-    {
+def protections(self):
-        "method": "CooldownPeriod",
+    return  [
-        "stop_duration_candles": 2
+        {
-    }
+            "method": "CooldownPeriod",
-]
+            "stop_duration_candles": 2
+        }
+    ]
 ```
 
 !!! Note
@@ -136,39 +148,42 @@ from freqtrade.strategy import IStrategy
 
 class AwesomeStrategy(IStrategy)
     timeframe = '1h'
-    protections = [
+    
-        {
+    @property
-            "method": "CooldownPeriod",
+    def protections(self):
-            "stop_duration_candles": 5
+        return [
-        },
+            {
-        {
+                "method": "CooldownPeriod",
-            "method": "MaxDrawdown",
+                "stop_duration_candles": 5
-            "lookback_period_candles": 48,
+            },
-            "trade_limit": 20,
+            {
-            "stop_duration_candles": 4,
+                "method": "MaxDrawdown",
-            "max_allowed_drawdown": 0.2
+                "lookback_period_candles": 48,
-        },
+                "trade_limit": 20,
-        {
+                "stop_duration_candles": 4,
-            "method": "StoplossGuard",
+                "max_allowed_drawdown": 0.2
-            "lookback_period_candles": 24,
+            },
-            "trade_limit": 4,
+            {
-            "stop_duration_candles": 2,
+                "method": "StoplossGuard",
-            "only_per_pair": False
+                "lookback_period_candles": 24,
-        },
+                "trade_limit": 4,
-        {
+                "stop_duration_candles": 2,
-            "method": "LowProfitPairs",
+                "only_per_pair": False
-            "lookback_period_candles": 6,
+            },
-            "trade_limit": 2,
+            {
-            "stop_duration_candles": 60,
+                "method": "LowProfitPairs",
-            "required_profit": 0.02
+                "lookback_period_candles": 6,
-        },
+                "trade_limit": 2,
-        {
+                "stop_duration_candles": 60,
-            "method": "LowProfitPairs",
+                "required_profit": 0.02
-            "lookback_period_candles": 24,
+            },
-            "trade_limit": 4,
+            {
-            "stop_duration_candles": 2,
+                "method": "LowProfitPairs",
-            "required_profit": 0.01
+                "lookback_period_candles": 24,
-        }
+                "trade_limit": 4,
-    ]
+                "stop_duration_candles": 2,
+                "required_profit": 0.01
+            }
+        ]
     # ...
 ```

docs/index.md

@@ -36,7 +36,7 @@ Freqtrade is a crypto-currency algorithmic trading software developed in python
 
 Please read the [exchange specific notes](exchanges.md) to learn about eventual, special configurations needed for each exchange.
 
-- [X] [Binance](https://www.binance.com/) ([*Note for binance users](exchanges.md#blacklists))
+- [X] [Binance](https://www.binance.com/) ([*Note for binance users](docs/exchanges.md#binance-blacklist))
 - [X] [Bittrex](https://bittrex.com/)
 - [X] [FTX](https://ftx.com)
 - [X] [Kraken](https://kraken.com/)
@@ -47,7 +47,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual,
 Exchanges confirmed working by the community:
 
 - [X] [Bitvavo](https://bitvavo.com/)
-- [X] [Kukoin](https://www.kucoin.com/)
+- [X] [Kucoin](https://www.kucoin.com/)
 
 ## Requirements
 

docs/requirements-docs.txt

@@ -1,4 +1,4 @@
 mkdocs==1.2.2
-mkdocs-material==7.2.1
+mkdocs-material==7.2.4
 mdx_truly_sane_lists==1.2
 pymdown-extensions==8.2

docs/sql_cheatsheet.md

@@ -110,7 +110,7 @@ DELETE FROM trades WHERE id = 31;
 Freqtrade supports PostgreSQL by using SQLAlchemy, which supports multiple different database systems.
 
 Installation:
-`pip install psycopg2`
+`pip install psycopg2-binary`
 
 Usage:
 `... --db-url postgresql+psycopg2://<username>:<password>@localhost:5432/<database>`

docs/strategy-advanced.md

@@ -114,6 +114,36 @@ class AwesomeStrategy(IStrategy):
 
 See [Dataframe access](#dataframe-access) for more information about dataframe use in strategy callbacks.
 
+## Buy Tag
+
+When your strategy has multiple buy signals, you can name the signal that triggered.
+Then you can access you buy signal on `custom_sell`
+
+```python
+def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+    dataframe.loc[
+        (
+            (dataframe['rsi'] < 35) &
+            (dataframe['volume'] > 0)
+        ),
+        ['buy', 'buy_tag']] = (1, 'buy_signal_rsi')
+
+    return dataframe
+
+def custom_sell(self, pair: str, trade: Trade, current_time: datetime, current_rate: float,
+                    current_profit: float, **kwargs):
+    dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
+    last_candle = dataframe.iloc[-1].squeeze()
+    if trade.buy_tag == 'buy_signal_rsi' and last_candle['rsi'] > 80:
+        return 'sell_signal_rsi'
+    return None
+
+```
+
+!!! Note
+    `buy_tag` is limited to 100 characters, remaining data will be truncated.
+
+
 ## Custom stoploss
 
 The stoploss price can only ever move upwards - if the stoploss value returned from `custom_stoploss` would result in a lower stoploss price than was previously set, it will be ignored. The traditional `stoploss` value serves as an absolute lower level and will be instated as the initial stoploss.
@@ -327,6 +357,55 @@ See [Dataframe access](#dataframe-access) for more information about dataframe u
 
 ---
 
+## Custom order price rules
+
+By default, freqtrade use the orderbook to automatically set an order price([Relevant documentation](configuration.md#prices-used-for-orders)), you also have the option to create custom order prices based on your strategy.
+
+You can use this feature by creating a `custom_entry_price()` function in your strategy file to customize entry prices and `custom_exit_price()` for exits.
+
+!!! Note
+    If your custom pricing function return None or an invalid value, price will fall back to `proposed_rate`, which is based on the regular pricing configuration.
+
+### Custom order entry and exit price example
+
+``` python
+from datetime import datetime, timedelta, timezone
+from freqtrade.persistence import Trade
+
+class AwesomeStrategy(IStrategy):
+
+    # ... populate_* methods
+
+    def custom_entry_price(self, pair: str, current_time: datetime,
+                           proposed_rate, **kwargs) -> float:
+
+        dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair,
+                                                                timeframe=self.timeframe)
+        new_entryprice = dataframe['bollinger_10_lowerband'].iat[-1]
+        
+        return new_entryprice
+
+    def custom_exit_price(self, pair: str, trade: Trade,
+                          current_time: datetime, proposed_rate: float,
+                          current_profit: float, **kwargs) -> float:
+
+        dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair,
+                                                                timeframe=self.timeframe)
+        new_exitprice = dataframe['bollinger_10_upperband'].iat[-1]
+        
+        return new_exitprice
+
+```
+
+!!! Warning
+    Modifying entry and exit prices will only work for limit orders. Depending on the price chosen, this can result in a lot of unfilled orders. By default the maximum allowed distance between the current price and the custom price is 2%, this value can be changed in config with the `custom_price_max_distance_ratio` parameter.
+
+!!! Example
+    If the new_entryprice is 97, the proposed_rate is 100 and the `custom_price_max_distance_ratio` is set to 2%, The retained valid custom entry price will be 98.
+
+!!! Warning "No backtesting support"
+    Custom entry-prices are currently not supported during backtesting.
+
 ## Custom order timeout rules
 
 Simple, time-based order-timeouts can be configured either via strategy or in the configuration in the `unfilledtimeout` section.

docs/strategy_analysis_example.md

@@ -228,7 +228,7 @@ graph = generate_candlestick_graph(pair=pair,
 # Show graph inline
 # graph.show()
 
-# Render graph in a seperate window
+# Render graph in a separate window
 graph.show(renderer="browser")
 
 ```

docs/utils.md

@@ -627,7 +627,7 @@ FreqUI will also show the backtesting results.
 
 ```
 usage: freqtrade webserver [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
-                           [--userdir PATH] [-s NAME] [--strategy-path PATH]
+                           [--userdir PATH]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -648,12 +648,6 @@ Common arguments:
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 
-Strategy arguments:
-  -s NAME, --strategy NAME
-                        Specify strategy class name which will be used by the
-                        bot.
-  --strategy-path PATH  Specify additional strategy lookup path.
-
 ```
 
 ## List Hyperopt results

docs/webhook-config.md

@@ -83,6 +83,7 @@ Possible parameters are:
 * `fiat_currency`
 * `order_type`
 * `current_rate`
+* `buy_tag`
 
 ### Webhookbuycancel
 
@@ -100,6 +101,7 @@ Possible parameters are:
 * `fiat_currency`
 * `order_type`
 * `current_rate`
+* `buy_tag`
 
 ### Webhookbuyfill
 
@@ -115,6 +117,7 @@ Possible parameters are:
 * `stake_amount`
 * `stake_currency`
 * `fiat_currency`
+* `buy_tag`
 
 ### Webhooksell
 

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2021.7'
+__version__ = '2021.8'
 
 if __version__ == 'develop':
 

freqtrade/commands/build_config_commands.py

@@ -193,7 +193,7 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None:
         selections['exchange'] = render_template(
             templatefile=f"subtemplates/exchange_{exchange_template}.j2",
             arguments=selections
-            )
+        )
     except TemplateNotFound:
         selections['exchange'] = render_template(
             templatefile="subtemplates/exchange_generic.j2",

freqtrade/commands/cli_options.py

@@ -162,7 +162,7 @@ AVAILABLE_CLI_OPTIONS = {
         'Please note that ticker-interval needs to be set either in config '
         'or via command line. When using this together with `--export trades`, '
         'the strategy-name is injected into the filename '
-        '(so `backtest-data.json` becomes `backtest-data-DefaultStrategy.json`',
+        '(so `backtest-data.json` becomes `backtest-data-SampleStrategy.json`',
         nargs='+',
     ),
     "export": Arg(
@@ -218,7 +218,7 @@ AVAILABLE_CLI_OPTIONS = {
     "spaces": Arg(
         '--spaces',
         help='Specify which parameters to hyperopt. Space-separated list.',
-        choices=['all', 'buy', 'sell', 'roi', 'stoploss', 'trailing', 'default'],
+        choices=['all', 'buy', 'sell', 'roi', 'stoploss', 'trailing', 'protection', 'default'],
         nargs='+',
         default='default',
     ),

freqtrade/commands/deploy_commands.py

@@ -38,15 +38,15 @@ def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: st
     indicators = render_template_with_fallback(
         templatefile=f"subtemplates/indicators_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/indicators_{fallback}.j2",
-        )
+    )
     buy_trend = render_template_with_fallback(
         templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/buy_trend_{fallback}.j2",
-        )
+    )
     sell_trend = render_template_with_fallback(
         templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/sell_trend_{fallback}.j2",
-        )
+    )
     plot_config = render_template_with_fallback(
         templatefile=f"subtemplates/plot_config_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/plot_config_{fallback}.j2",
@@ -74,8 +74,6 @@ def start_new_strategy(args: Dict[str, Any]) -> None:
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
     if "strategy" in args and args["strategy"]:
-        if args["strategy"] == "DefaultStrategy":
-            raise OperationalException("DefaultStrategy is not allowed as name.")
 
         new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py')
 
@@ -97,19 +95,19 @@ def deploy_new_hyperopt(hyperopt_name: str, hyperopt_path: Path, subtemplate: st
     buy_guards = render_template_with_fallback(
         templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/hyperopt_buy_guards_{fallback}.j2",
-        )
+    )
     sell_guards = render_template_with_fallback(
         templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/hyperopt_sell_guards_{fallback}.j2",
-        )
+    )
     buy_space = render_template_with_fallback(
         templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/hyperopt_buy_space_{fallback}.j2",
-        )
+    )
     sell_space = render_template_with_fallback(
         templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/hyperopt_sell_space_{fallback}.j2",
-        )
+    )
 
     strategy_text = render_template(templatefile='base_hyperopt.py.j2',
                                     arguments={"hyperopt": hyperopt_name,
@@ -128,8 +126,6 @@ def start_new_hyperopt(args: Dict[str, Any]) -> None:
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
     if 'hyperopt' in args and args['hyperopt']:
-        if args['hyperopt'] == 'DefaultHyperopt':
-            raise OperationalException("DefaultHyperopt is not allowed as name.")
 
         new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args['hyperopt'] + '.py')
 

freqtrade/commands/hyperopt_commands.py

@@ -1,6 +1,6 @@
 import logging
 from operator import itemgetter
-from typing import Any, Dict, List
+from typing import Any, Dict
 
 from colorama import init as colorama_init
 
@@ -28,30 +28,12 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     no_details = config.get('hyperopt_list_no_details', False)
     no_header = False
 
-    filteroptions = {
-        'only_best': config.get('hyperopt_list_best', False),
-        'only_profitable': config.get('hyperopt_list_profitable', False),
-        'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
-        'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
-        'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
-        'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
-        'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
-        'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
-        'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
-        'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
-        'filter_min_objective': config.get('hyperopt_list_min_objective', None),
-        'filter_max_objective': config.get('hyperopt_list_max_objective', None),
-    }
-
     results_file = get_latest_hyperopt_file(
         config['user_data_dir'] / 'hyperopt_results',
         config.get('hyperoptexportfilename'))
 
     # Previous evaluations
-    epochs = HyperoptTools.load_previous_results(results_file)
+    epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
-    total_epochs = len(epochs)
-
-    epochs = hyperopt_filter_epochs(epochs, filteroptions)
 
     if print_colorized:
         colorama_init(autoreset=True)
@@ -59,7 +41,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     if not export_csv:
         try:
             print(HyperoptTools.get_result_table(config, epochs, total_epochs,
-                                                 not filteroptions['only_best'],
+                                                 not config.get('hyperopt_list_best', False),
                                                  print_colorized, 0))
         except KeyboardInterrupt:
             print('User interrupted..')
@@ -71,7 +53,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
 
     if epochs and export_csv:
         HyperoptTools.export_csv_file(
-            config, epochs, total_epochs, not filteroptions['only_best'], export_csv
+            config, epochs, total_epochs, not config.get('hyperopt_list_best', False), export_csv
         )
 
 
@@ -91,26 +73,9 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
 
     n = config.get('hyperopt_show_index', -1)
 
-    filteroptions = {
-        'only_best': config.get('hyperopt_list_best', False),
-        'only_profitable': config.get('hyperopt_list_profitable', False),
-        'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
-        'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
-        'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
-        'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
-        'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
-        'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
-        'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
-        'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
-        'filter_min_objective': config.get('hyperopt_list_min_objective', None),
-        'filter_max_objective': config.get('hyperopt_list_max_objective', None)
-    }
-
     # Previous evaluations
-    epochs = HyperoptTools.load_previous_results(results_file)
+    epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
-    total_epochs = len(epochs)
 
-    epochs = hyperopt_filter_epochs(epochs, filteroptions)
     filtered_epochs = len(epochs)
 
     if n > filtered_epochs:
@@ -137,138 +102,3 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
 
         HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header,
                                          header_str="Epoch details")
-
-
-def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
-    """
-    Filter our items from the list of hyperopt results
-    TODO: after 2021.5 remove all "legacy" mode queries.
-    """
-    if filteroptions['only_best']:
-        epochs = [x for x in epochs if x['is_best']]
-    if filteroptions['only_profitable']:
-        epochs = [x for x in epochs if x['results_metrics'].get(
-            'profit', x['results_metrics'].get('profit_total', 0)) > 0]
-
-    epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
-
-    epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions)
-
-    epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions)
-
-    epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions)
-
-    logger.info(f"{len(epochs)} " +
-                ("best " if filteroptions['only_best'] else "") +
-                ("profitable " if filteroptions['only_profitable'] else "") +
-                "epochs found.")
-    return epochs
-
-
-def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int):
-    """
-    Filter epochs with trade-counts > trades
-    """
-    return [
-        x for x in epochs
-        if x['results_metrics'].get(
-            'trade_count', x['results_metrics'].get('total_trades', 0)
-        ) > trade_count
-    ]
-
-
-def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
-
-    if filteroptions['filter_min_trades'] > 0:
-        epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades'])
-
-    if filteroptions['filter_max_trades'] > 0:
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'trade_count', x['results_metrics'].get('total_trades')
-                ) < filteroptions['filter_max_trades']
-        ]
-    return epochs
-
-
-def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
-
-    def get_duration_value(x):
-        # Duration in minutes ...
-        if 'duration' in x['results_metrics']:
-            return x['results_metrics']['duration']
-        else:
-            # New mode
-            if 'holding_avg_s' in x['results_metrics']:
-                avg = x['results_metrics']['holding_avg_s']
-                return avg // 60
-            raise OperationalException(
-                "Holding-average not available. Please omit the filter on average time, "
-                "or rerun hyperopt with this version")
-
-    if filteroptions['filter_min_avg_time'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if get_duration_value(x) > filteroptions['filter_min_avg_time']
-        ]
-    if filteroptions['filter_max_avg_time'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if get_duration_value(x) < filteroptions['filter_max_avg_time']
-        ]
-
-    return epochs
-
-
-def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
-
-    if filteroptions['filter_min_avg_profit'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
-            ) > filteroptions['filter_min_avg_profit']
-        ]
-    if filteroptions['filter_max_avg_profit'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
-                ) < filteroptions['filter_max_avg_profit']
-        ]
-    if filteroptions['filter_min_total_profit'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'profit', x['results_metrics'].get('profit_total_abs', 0)
-                ) > filteroptions['filter_min_total_profit']
-        ]
-    if filteroptions['filter_max_total_profit'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'profit', x['results_metrics'].get('profit_total_abs', 0)
-                ) < filteroptions['filter_max_total_profit']
-        ]
-    return epochs
-
-
-def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
-
-    if filteroptions['filter_min_objective'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-
-        epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
-    if filteroptions['filter_max_objective'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-
-        epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
-
-    return epochs

freqtrade/configuration/check_exchange.py

@@ -51,10 +51,10 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
 
     if not is_exchange_known_ccxt(exchange):
         raise OperationalException(
-                f'Exchange "{exchange}" is not known to the ccxt library '
+            f'Exchange "{exchange}" is not known to the ccxt library '
-                f'and therefore not available for the bot.\n'
+            f'and therefore not available for the bot.\n'
-                f'The following exchanges are available for Freqtrade: '
+            f'The following exchanges are available for Freqtrade: '
-                f'{", ".join(available_exchanges())}'
+            f'{", ".join(available_exchanges())}'
         )
 
     valid, reason = validate_exchange(exchange)

freqtrade/configuration/config_validation.py

@@ -115,7 +115,7 @@ def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None:
     if conf.get('stoploss') == 0.0:
         raise OperationalException(
             'The config stoploss needs to be different from 0 to avoid problems with sell orders.'
-            )
+        )
     # Skip if trailing stoploss is not activated
     if not conf.get('trailing_stop', False):
         return
@@ -180,7 +180,7 @@ def _validate_protections(conf: Dict[str, Any]) -> None:
             raise OperationalException(
                 "Protections must specify either `stop_duration` or `stop_duration_candles`.\n"
                 f"Please fix the protection {prot.get('method')}"
-                )
+            )
 
         if ('lookback_period' in prot and 'lookback_period_candles' in prot):
             raise OperationalException(

freqtrade/configuration/configuration.py

@@ -11,6 +11,7 @@ from freqtrade import constants
 from freqtrade.configuration.check_exchange import check_exchange
 from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
 from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
+from freqtrade.configuration.environment_vars import enironment_vars_to_dict
 from freqtrade.configuration.load_config import load_config_file, load_file
 from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode
 from freqtrade.exceptions import OperationalException
@@ -71,6 +72,11 @@ class Configuration:
 
             # Merge config options, overwriting old values
             config = deep_merge_dicts(load_config_file(path), config)
+
+        # Load environment variables
+        env_data = enironment_vars_to_dict()
+        config = deep_merge_dicts(env_data, config)
+
         config['config_files'] = files
         # Normalize config
         if 'internals' not in config:

freqtrade/configuration/deprecated_settings.py

@@ -108,5 +108,8 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
             raise OperationalException(
                 "Both 'timeframe' and 'ticker_interval' detected."
                 "Please remove 'ticker_interval' from your configuration to continue operating."
-                )
+            )
         config['timeframe'] = config['ticker_interval']
+
+    if 'protections' in config:
+        logger.warning("DEPRECATED: Setting 'protections' in the configuration is deprecated.")

freqtrade/configuration/environment_vars.py

@@ -0,0 +1,54 @@
+import logging
+import os
+from typing import Any, Dict
+
+from freqtrade.constants import ENV_VAR_PREFIX
+from freqtrade.misc import deep_merge_dicts
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_var_typed(val):
+    try:
+        return int(val)
+    except ValueError:
+        try:
+            return float(val)
+        except ValueError:
+            if val.lower() in ('t', 'true'):
+                return True
+            elif val.lower() in ('f', 'false'):
+                return False
+    # keep as string
+    return val
+
+
+def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]:
+    """
+    Environment variables must be prefixed with FREQTRADE.
+    FREQTRADE__{section}__{key}
+    :param env_dict: Dictionary to validate - usually os.environ
+    :param prefix: Prefix to consider (usually FREQTRADE__)
+    :return: Nested dict based on available and relevant variables.
+    """
+    relevant_vars: Dict[str, Any] = {}
+
+    for env_var, val in sorted(env_dict.items()):
+        if env_var.startswith(prefix):
+            logger.info(f"Loading variable '{env_var}'")
+            key = env_var.replace(prefix, '')
+            for k in reversed(key.split('__')):
+                val = {k.lower(): get_var_typed(val) if type(val) != dict else val}
+            relevant_vars = deep_merge_dicts(val, relevant_vars)
+
+    return relevant_vars
+
+
+def enironment_vars_to_dict() -> Dict[str, Any]:
+    """
+    Read environment variables and return a nested dict for relevant variables
+    Relevant variables must follow the FREQTRADE__{section}__{key} pattern
+    :return: Nested dict based on available and relevant variables.
+    """
+    return flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX)

freqtrade/constants.py

@@ -47,6 +47,9 @@ USERPATH_STRATEGIES = 'strategies'
 USERPATH_NOTEBOOKS = 'notebooks'
 
 TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent']
+ENV_VAR_PREFIX = 'FREQTRADE__'
+
+NON_OPEN_EXCHANGE_STATES = ('cancelled', 'canceled', 'closed', 'expired')
 
 
 # Define decimals per coin for outputs
@@ -190,6 +193,9 @@ CONF_SCHEMA = {
             },
             'required': ['price_side']
         },
+        'custom_price_max_distance_ratio': {
+           'type': 'number', 'minimum': 0.0
+        },
         'order_types': {
             'type': 'object',
             'properties': {
@@ -279,7 +285,7 @@ CONF_SCHEMA = {
                             'type': 'string',
                             'enum': TELEGRAM_SETTING_OPTIONS,
                             'default': 'off'
-                            },
+                        },
                     }
                 },
                 'reload': {'type': 'boolean'},

freqtrade/data/btanalysis.py

@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
 BT_DATA_COLUMNS_OLD = ["pair", "profit_percent", "open_date", "close_date", "index",
                        "trade_duration", "open_rate", "close_rate", "open_at_end", "sell_reason"]
 
-# Mid-term format, crated by BacktestResult Named Tuple
+# Mid-term format, created by BacktestResult Named Tuple
 BT_DATA_COLUMNS_MID = ['pair', 'profit_percent', 'open_date', 'close_date', 'trade_duration',
                        'open_rate', 'close_rate', 'open_at_end', 'sell_reason', 'fee_open',
                        'fee_close', 'amount', 'profit_abs', 'profit_ratio']
@@ -30,7 +30,7 @@ BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
                    'fee_open', 'fee_close', 'trade_duration',
                    'profit_ratio', 'profit_abs', 'sell_reason',
                    'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
-                   'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', ]
+                   'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'buy_tag']
 
 
 def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str:

freqtrade/data/converter.py

@@ -242,7 +242,7 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to:
     :param config: Config dictionary
     :param convert_from: Source format
     :param convert_to: Target format
-    :param erase: Erase souce data (does not apply if source and target format are identical)
+    :param erase: Erase source data (does not apply if source and target format are identical)
     """
     from freqtrade.data.history.idatahandler import get_datahandler
     src = get_datahandler(config['datadir'], convert_from)
@@ -267,7 +267,7 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to:
     :param config: Config dictionary
     :param convert_from: Source format
     :param convert_to: Target format
-    :param erase: Erase souce data (does not apply if source and target format are identical)
+    :param erase: Erase source data (does not apply if source and target format are identical)
     """
     from freqtrade.data.history.idatahandler import get_datahandler
     src = get_datahandler(config['datadir'], convert_from)

freqtrade/data/dataprovider.py

@@ -10,11 +10,12 @@ from typing import Any, Dict, List, Optional, Tuple
 
 from pandas import DataFrame
 
+from freqtrade.configuration import TimeRange
 from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe
 from freqtrade.data.history import load_pair_history
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import ExchangeError, OperationalException
-from freqtrade.exchange import Exchange
+from freqtrade.exchange import Exchange, timeframe_to_seconds
 
 
 logger = logging.getLogger(__name__)
@@ -31,6 +32,7 @@ class DataProvider:
         self._pairlists = pairlists
         self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
         self.__slice_index: Optional[int] = None
+        self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {}
 
     def _set_dataframe_max_index(self, limit_index: int):
         """
@@ -62,11 +64,22 @@ class DataProvider:
         :param pair: pair to get the data for
         :param timeframe: timeframe to get data for
         """
-        return load_pair_history(pair=pair,
+        saved_pair = (pair, str(timeframe))
-                                 timeframe=timeframe or self._config['timeframe'],
+        if saved_pair not in self.__cached_pairs_backtesting:
-                                 datadir=self._config['datadir'],
+            timerange = TimeRange.parse_timerange(None if self._config.get(
-                                 data_format=self._config.get('dataformat_ohlcv', 'json')
+                'timerange') is None else str(self._config.get('timerange')))
-                                 )
+            # Move informative start time respecting startup_candle_count
+            timerange.subtract_start(
+                timeframe_to_seconds(str(timeframe)) * self._config.get('startup_candle_count', 0)
+            )
+            self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
+                pair=pair,
+                timeframe=timeframe or self._config['timeframe'],
+                datadir=self._config['datadir'],
+                timerange=timerange,
+                data_format=self._config.get('dataformat_ohlcv', 'json')
+            )
+        return self.__cached_pairs_backtesting[saved_pair].copy()
 
     def get_pair_dataframe(self, pair: str, timeframe: str = None) -> DataFrame:
         """

freqtrade/data/history/history_utils.py

@@ -117,10 +117,11 @@ def refresh_data(datadir: Path,
     :param timerange: Limit data to be loaded to this timerange
     """
     data_handler = get_datahandler(datadir, data_format)
-    for pair in pairs:
+    for idx, pair in enumerate(pairs):
-        _download_pair_history(pair=pair, timeframe=timeframe,
+        process = f'{idx}/{len(pairs)}'
-                               datadir=datadir, timerange=timerange,
+        _download_pair_history(pair=pair, process=process,
-                               exchange=exchange, data_handler=data_handler)
+                               timeframe=timeframe, datadir=datadir,
+                               timerange=timerange, exchange=exchange, data_handler=data_handler)
 
 
 def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange],
@@ -153,13 +154,14 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
     return data, start_ms
 
 
-def _download_pair_history(datadir: Path,
+def _download_pair_history(pair: str, *,
+                           datadir: Path,
                            exchange: Exchange,
-                           pair: str, *,
-                           new_pairs_days: int = 30,
                            timeframe: str = '5m',
-                           timerange: Optional[TimeRange] = None,
+                           process: str = '',
-                           data_handler: IDataHandler = None) -> bool:
+                           new_pairs_days: int = 30,
+                           data_handler: IDataHandler = None,
+                           timerange: Optional[TimeRange] = None) -> bool:
     """
     Download latest candles from the exchange for the pair and timeframe passed in parameters
     The data is downloaded starting from the last correct data that
@@ -177,7 +179,7 @@ def _download_pair_history(datadir: Path,
 
     try:
         logger.info(
-            f'Download history data for pair: "{pair}", timeframe: {timeframe} '
+            f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe} '
             f'and store in {datadir}.'
         )
 
@@ -234,7 +236,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
     """
     pairs_not_available = []
     data_handler = get_datahandler(datadir, data_format)
-    for pair in pairs:
+    for idx, pair in enumerate(pairs, start=1):
         if pair not in exchange.markets:
             pairs_not_available.append(pair)
             logger.info(f"Skipping pair {pair}...")
@@ -247,10 +249,11 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
                         f'Deleting existing data for pair {pair}, interval {timeframe}.')
 
             logger.info(f'Downloading pair {pair}, interval {timeframe}.')
-            _download_pair_history(datadir=datadir, exchange=exchange,
+            process = f'{idx}/{len(pairs)}'
-                                   pair=pair, timeframe=str(timeframe),
+            _download_pair_history(pair=pair, process=process,
-                                   new_pairs_days=new_pairs_days,
+                                   datadir=datadir, exchange=exchange,
-                                   timerange=timerange, data_handler=data_handler)
+                                   timerange=timerange, data_handler=data_handler,
+                                   timeframe=str(timeframe), new_pairs_days=new_pairs_days)
     return pairs_not_available
 
 

freqtrade/data/history/jsondatahandler.py

@@ -62,7 +62,7 @@ class JsonDataHandler(IDataHandler):
         filename = self._pair_data_filename(self._datadir, pair, timeframe)
         _data = data.copy()
         # Convert date to int
-        _data['date'] = _data['date'].astype(np.int64) // 1000 // 1000
+        _data['date'] = _data['date'].view(np.int64) // 1000 // 1000
 
         # Reset index, select only appropriate columns and save as json
         _data.reset_index(drop=True).loc[:, self._columns].to_json(

freqtrade/edge/edge_positioning.py

@@ -151,7 +151,7 @@ class Edge:
         # Fake run-mode to Edge
         prior_rm = self.config['runmode']
         self.config['runmode'] = RunMode.EDGE
-        preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
+        preprocessed = self.strategy.advise_all_indicators(data)
         self.config['runmode'] = prior_rm
 
         # Print timeframe
@@ -231,12 +231,12 @@ class Edge:
                     'Minimum expectancy and minimum winrate are met only for %s,'
                     ' so other pairs are filtered out.',
                     self._final_pairs
-                    )
+                )
             else:
                 logger.info(
                     'Edge removed all pairs as no pair with minimum expectancy '
                     'and minimum winrate was found !'
-                    )
+                )
 
         return self._final_pairs
 
@@ -247,7 +247,7 @@ class Edge:
         final = []
         for pair, info in self._cached_pairs.items():
             if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
-                 info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
+                    info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
                 final.append({
                     'Pair': pair,
                     'Winrate': info.winrate,

freqtrade/enums/__init__.py

@@ -3,5 +3,5 @@ from freqtrade.enums.backteststate import BacktestState
 from freqtrade.enums.rpcmessagetype import RPCMessageType
 from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode
 from freqtrade.enums.selltype import SellType
-from freqtrade.enums.signaltype import SignalType
+from freqtrade.enums.signaltype import SignalTagType, SignalType
 from freqtrade.enums.state import State

freqtrade/enums/signaltype.py

@@ -7,3 +7,10 @@ class SignalType(Enum):
     """
     BUY = "buy"
     SELL = "sell"
+
+
+class SignalTagType(Enum):
+    """
+    Enum for signal columns
+    """
+    BUY_TAG = "buy_tag"

freqtrade/exchange/__init__.py

@@ -15,6 +15,7 @@ from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges,
                                          timeframe_to_seconds, validate_exchange,
                                          validate_exchanges)
 from freqtrade.exchange.ftx import Ftx
+from freqtrade.exchange.gateio import Gateio
 from freqtrade.exchange.hitbtc import Hitbtc
 from freqtrade.exchange.kraken import Kraken
 from freqtrade.exchange.kucoin import Kucoin

freqtrade/exchange/exchange.py

@@ -19,7 +19,8 @@ from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRU
                                             decimal_to_precision)
 from pandas import DataFrame
 
-from freqtrade.constants import DEFAULT_AMOUNT_RESERVE_PERCENT, ListPairsWithTimeframes
+from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES,
+                                 ListPairsWithTimeframes)
 from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
 from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
                                   InvalidOrderException, OperationalException, PricingError,
@@ -618,6 +619,8 @@ class Exchange:
         if self.exchange_has('fetchL2OrderBook'):
             ob = self.fetch_l2_order_book(pair, 20)
             ob_type = 'asks' if side == 'buy' else 'bids'
+            slippage = 0.05
+            max_slippage_val = rate * ((1 + slippage) if side == 'buy' else (1 - slippage))
 
             remaining_amount = amount
             filled_amount = 0
@@ -626,7 +629,9 @@ class Exchange:
                 book_entry_coin_volume = book_entry[1]
                 if remaining_amount > 0:
                     if remaining_amount < book_entry_coin_volume:
+                        # Orderbook at this slot bigger than remaining amount
                         filled_amount += remaining_amount * book_entry_price
+                        break
                     else:
                         filled_amount += book_entry_coin_volume * book_entry_price
                     remaining_amount -= book_entry_coin_volume
@@ -635,7 +640,14 @@ class Exchange:
             else:
                 # If remaining_amount wasn't consumed completely (break was not called)
                 filled_amount += remaining_amount * book_entry_price
-            forecast_avg_filled_price = filled_amount / amount
+            forecast_avg_filled_price = max(filled_amount, 0) / amount
+            # Limit max. slippage to specified value
+            if side == 'buy':
+                forecast_avg_filled_price = min(forecast_avg_filled_price, max_slippage_val)
+
+            else:
+                forecast_avg_filled_price = max(forecast_avg_filled_price, max_slippage_val)
+
             return self.price_to_precision(pair, forecast_avg_filled_price)
 
         return rate
@@ -689,7 +701,16 @@ class Exchange:
     # Order handling
 
     def create_order(self, pair: str, ordertype: str, side: str, amount: float,
-                     rate: float, params: Dict = {}) -> Dict:
+                     rate: float, time_in_force: str = 'gtc') -> Dict:
+
+        if self._config['dry_run']:
+            dry_order = self.create_dry_run_order(pair, ordertype, side, amount, rate)
+            return dry_order
+
+        params = self._params.copy()
+        if time_in_force != 'gtc' and ordertype != 'market':
+            params.update({'timeInForce': time_in_force})
+
         try:
             # Set the precision for amount and price(rate) as accepted by the exchange
             amount = self.amount_to_precision(pair, amount)
@@ -720,32 +741,6 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
-    def buy(self, pair: str, ordertype: str, amount: float,
-            rate: float, time_in_force: str) -> Dict:
-
-        if self._config['dry_run']:
-            dry_order = self.create_dry_run_order(pair, ordertype, "buy", amount, rate)
-            return dry_order
-
-        params = self._params.copy()
-        if time_in_force != 'gtc' and ordertype != 'market':
-            params.update({'timeInForce': time_in_force})
-
-        return self.create_order(pair, ordertype, 'buy', amount, rate, params)
-
-    def sell(self, pair: str, ordertype: str, amount: float,
-             rate: float, time_in_force: str = 'gtc') -> Dict:
-
-        if self._config['dry_run']:
-            dry_order = self.create_dry_run_order(pair, ordertype, "sell", amount, rate)
-            return dry_order
-
-        params = self._params.copy()
-        if time_in_force != 'gtc' and ordertype != 'market':
-            params.update({'timeInForce': time_in_force})
-
-        return self.create_order(pair, ordertype, 'sell', amount, rate, params)
-
     def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool:
         """
         Verify stop_loss against stoploss-order value (limit or price)
@@ -810,7 +805,7 @@ class Exchange:
         :param order: Order dict as returned from fetch_order()
         :return: True if order has been cancelled without being filled, False otherwise.
         """
-        return (order.get('status') in ('closed', 'canceled', 'cancelled')
+        return (order.get('status') in NON_OPEN_EXCHANGE_STATES
                 and order.get('filled') == 0.0)
 
     @retrier
@@ -1044,7 +1039,7 @@ class Exchange:
             logger.debug(f"Using Last {conf_strategy['price_side'].capitalize()} / Last Price")
             ticker = self.fetch_ticker(pair)
             ticker_rate = ticker[conf_strategy['price_side']]
-            if ticker['last']:
+            if ticker['last'] and ticker_rate:
                 if side == 'buy' and ticker_rate > ticker['last']:
                     balance = conf_strategy['ask_last_balance']
                     ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate)
@@ -1259,7 +1254,7 @@ class Exchange:
         logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list))
 
         input_coroutines = []
-
+        cached_pairs = []
         # Gather coroutines to run
         for pair, timeframe in set(pair_list):
             if (((pair, timeframe) not in self._klines)
@@ -1271,6 +1266,7 @@ class Exchange:
                     "Using cached candle (OHLCV) data for pair %s, timeframe %s ...",
                     pair, timeframe
                 )
+                cached_pairs.append((pair, timeframe))
 
         results = asyncio.get_event_loop().run_until_complete(
             asyncio.gather(*input_coroutines, return_exceptions=True))
@@ -1293,6 +1289,10 @@ class Exchange:
             results_df[(pair, timeframe)] = ohlcv_df
             if cache:
                 self._klines[(pair, timeframe)] = ohlcv_df
+        # Return cached klines
+        for pair, timeframe in cached_pairs:
+            results_df[(pair, timeframe)] = self.klines((pair, timeframe), copy=False)
+
         return results_df
 
     def _now_is_time_to_refresh(self, pair: str, timeframe: str) -> bool:
@@ -1503,7 +1503,7 @@ class Exchange:
         :returns List of trade data
         """
         if not self.exchange_has("fetchTrades"):
-            raise OperationalException("This exchange does not suport downloading Trades.")
+            raise OperationalException("This exchange does not support downloading Trades.")
 
         return asyncio.get_event_loop().run_until_complete(
             self._async_get_trade_history(pair=pair, since=since,

freqtrade/exchange/gateio.py

@@ -0,0 +1,23 @@
+""" Gate.io exchange subclass """
+import logging
+from typing import Dict
+
+from freqtrade.exchange import Exchange
+
+
+logger = logging.getLogger(__name__)
+
+
+class Gateio(Exchange):
+    """
+    Gate.io exchange class. Contains adjustments needed for Freqtrade to work
+    with this exchange.
+
+    Please note that this exchange is not included in the list of exchanges
+    officially supported by the Freqtrade development team. So some features
+    may still not work as expected.
+    """
+
+    _ft_has: Dict = {
+        "ohlcv_candle_limit": 1000,
+    }

freqtrade/freqtradebot.py

@@ -420,20 +420,24 @@ class FreqtradeBot(LoggingMixin):
             return False
 
         # running get_signal on historical data fetched
-        (buy, sell) = self.strategy.get_signal(pair, self.strategy.timeframe, analyzed_df)
+        (buy, sell, buy_tag) = self.strategy.get_signal(
+            pair,
+            self.strategy.timeframe,
+            analyzed_df
+        )
 
         if buy and not sell:
             stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge)
 
             bid_check_dom = self.config.get('bid_strategy', {}).get('check_depth_of_market', {})
             if ((bid_check_dom.get('enabled', False)) and
-               (bid_check_dom.get('bids_to_ask_delta', 0) > 0)):
+                    (bid_check_dom.get('bids_to_ask_delta', 0) > 0)):
                 if self._check_depth_of_market_buy(pair, bid_check_dom):
-                    return self.execute_buy(pair, stake_amount)
+                    return self.execute_entry(pair, stake_amount, buy_tag=buy_tag)
                 else:
                     return False
 
-            return self.execute_buy(pair, stake_amount)
+            return self.execute_entry(pair, stake_amount, buy_tag=buy_tag)
         else:
             return False
 
@@ -461,8 +465,8 @@ class FreqtradeBot(LoggingMixin):
             logger.info(f"Bids to asks delta for {pair} does not satisfy condition.")
             return False
 
-    def execute_buy(self, pair: str, stake_amount: float, price: Optional[float] = None,
+    def execute_entry(self, pair: str, stake_amount: float, price: Optional[float] = None,
-                    forcebuy: bool = False) -> bool:
+                      forcebuy: bool = False, buy_tag: Optional[str] = None) -> bool:
         """
         Executes a limit buy for the given pair
         :param pair: pair for which we want to create a LIMIT_BUY
@@ -475,7 +479,13 @@ class FreqtradeBot(LoggingMixin):
             buy_limit_requested = price
         else:
             # Calculate price
-            buy_limit_requested = self.exchange.get_rate(pair, refresh=True, side="buy")
+            proposed_buy_rate = self.exchange.get_rate(pair, refresh=True, side="buy")
+            custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price,
+                                                       default_retval=proposed_buy_rate)(
+                pair=pair, current_time=datetime.now(timezone.utc),
+                proposed_rate=proposed_buy_rate)
+
+            buy_limit_requested = self.get_valid_price(custom_entry_price, proposed_buy_rate)
 
         if not buy_limit_requested:
             raise PricingError('Could not determine buy price.')
@@ -510,9 +520,9 @@ class FreqtradeBot(LoggingMixin):
             logger.info(f"User requested abortion of buying {pair}")
             return False
         amount = self.exchange.amount_to_precision(pair, amount)
-        order = self.exchange.buy(pair=pair, ordertype=order_type,
+        order = self.exchange.create_order(pair=pair, ordertype=order_type, side="buy",
-                                  amount=amount, rate=buy_limit_requested,
+                                           amount=amount, rate=buy_limit_requested,
-                                  time_in_force=time_in_force)
+                                           time_in_force=time_in_force)
         order_obj = Order.parse_from_ccxt_object(order, pair, 'buy')
         order_id = order['id']
         order_status = order.get('status', None)
@@ -565,6 +575,7 @@ class FreqtradeBot(LoggingMixin):
             exchange=self.exchange.id,
             open_order_id=order_id,
             strategy=self.strategy.get_strategy_name(),
+            buy_tag=buy_tag,
             timeframe=timeframe_to_minutes(self.config['timeframe'])
         )
         trade.orders.append(order_obj)
@@ -590,6 +601,7 @@ class FreqtradeBot(LoggingMixin):
         msg = {
             'trade_id': trade.id,
             'type': RPCMessageType.BUY,
+            'buy_tag': trade.buy_tag,
             'exchange': self.exchange.name.capitalize(),
             'pair': trade.pair,
             'limit': trade.open_rate,
@@ -614,6 +626,7 @@ class FreqtradeBot(LoggingMixin):
         msg = {
             'trade_id': trade.id,
             'type': RPCMessageType.BUY_CANCEL,
+            'buy_tag': trade.buy_tag,
             'exchange': self.exchange.name.capitalize(),
             'pair': trade.pair,
             'limit': trade.open_rate,
@@ -634,6 +647,7 @@ class FreqtradeBot(LoggingMixin):
         msg = {
             'trade_id': trade.id,
             'type': RPCMessageType.BUY_FILL,
+            'buy_tag': trade.buy_tag,
             'exchange': self.exchange.name.capitalize(),
             'pair': trade.pair,
             'open_rate': trade.open_rate,
@@ -692,7 +706,11 @@ class FreqtradeBot(LoggingMixin):
             analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair,
                                                                       self.strategy.timeframe)
 
-            (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.timeframe, analyzed_df)
+            (buy, sell, _) = self.strategy.get_signal(
+                trade.pair,
+                self.strategy.timeframe,
+                analyzed_df
+            )
 
         logger.debug('checking sell')
         sell_rate = self.exchange.get_rate(trade.pair, refresh=True, side="sell")
@@ -727,7 +745,7 @@ class FreqtradeBot(LoggingMixin):
             trade.stoploss_order_id = None
             logger.error(f'Unable to place a stoploss order on exchange. {e}')
             logger.warning('Selling the trade forcefully')
-            self.execute_sell(trade, trade.stop_loss, sell_reason=SellCheckTuple(
+            self.execute_trade_exit(trade, trade.stop_loss, sell_reason=SellCheckTuple(
                 sell_type=SellType.EMERGENCY_SELL))
 
         except ExchangeError:
@@ -845,7 +863,7 @@ class FreqtradeBot(LoggingMixin):
 
         if should_sell.sell_flag:
             logger.info(f'Executing Sell for {trade.pair}. Reason: {should_sell.sell_type}')
-            self.execute_sell(trade, sell_rate, should_sell)
+            self.execute_trade_exit(trade, sell_rate, should_sell)
             return True
         return False
 
@@ -927,7 +945,7 @@ class FreqtradeBot(LoggingMixin):
         was_trade_fully_canceled = False
 
         # Cancelled orders may have the status of 'canceled' or 'closed'
-        if order['status'] not in ('cancelled', 'canceled', 'closed'):
+        if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
             filled_val = order.get('filled', 0.0) or 0.0
             filled_stake = filled_val * trade.open_rate
             minstake = self.exchange.get_min_pair_stake_amount(
@@ -943,7 +961,7 @@ class FreqtradeBot(LoggingMixin):
             # Avoid race condition where the order could not be cancelled coz its already filled.
             # Simply bailing here is the only safe way - as this order will then be
             # handled in the next iteration.
-            if corder.get('status') not in ('cancelled', 'canceled', 'closed'):
+            if corder.get('status') not in constants.NON_OPEN_EXCHANGE_STATES:
                 logger.warning(f"Order {trade.open_order_id} for {trade.pair} not cancelled.")
                 return False
         else:
@@ -965,7 +983,7 @@ class FreqtradeBot(LoggingMixin):
             # if trade is partially complete, edit the stake details for the trade
             # and close the order
             # cancel_order may not contain the full order dict, so we need to fallback
-            # to the order dict aquired before cancelling.
+            # to the order dict acquired before cancelling.
             # we need to fall back to the values from order if corder does not contain these keys.
             trade.amount = filled_amount
             trade.stake_amount = trade.amount * trade.open_rate
@@ -1046,9 +1064,9 @@ class FreqtradeBot(LoggingMixin):
             raise DependencyException(
                 f"Not enough amount to sell. Trade-amount: {amount}, Wallet: {wallet_amount}")
 
-    def execute_sell(self, trade: Trade, limit: float, sell_reason: SellCheckTuple) -> bool:
+    def execute_trade_exit(self, trade: Trade, limit: float, sell_reason: SellCheckTuple) -> bool:
         """
-        Executes a limit sell for the given trade and limit
+        Executes a trade exit for the given trade and limit
         :param trade: Trade instance
         :param limit: limit rate for the sell order
         :param sell_reason: Reason the sell was triggered
@@ -1064,6 +1082,17 @@ class FreqtradeBot(LoggingMixin):
            and self.strategy.order_types['stoploss_on_exchange']:
             limit = trade.stop_loss
 
+        # set custom_exit_price if available
+        proposed_limit_rate = limit
+        current_profit = trade.calc_profit_ratio(limit)
+        custom_exit_price = strategy_safe_wrapper(self.strategy.custom_exit_price,
+                                                  default_retval=proposed_limit_rate)(
+            pair=trade.pair, trade=trade,
+            current_time=datetime.now(timezone.utc),
+            proposed_rate=proposed_limit_rate, current_profit=current_profit)
+
+        limit = self.get_valid_price(custom_exit_price, proposed_limit_rate)
+
         # First cancelling stoploss on exchange ...
         if self.strategy.order_types.get('stoploss_on_exchange') and trade.stoploss_order_id:
             try:
@@ -1094,11 +1123,11 @@ class FreqtradeBot(LoggingMixin):
 
         try:
             # Execute sell and update trade record
-            order = self.exchange.sell(pair=trade.pair,
+            order = self.exchange.create_order(pair=trade.pair,
-                                       ordertype=order_type,
+                                               ordertype=order_type, side="sell",
-                                       amount=amount, rate=limit,
+                                               amount=amount, rate=limit,
-                                       time_in_force=time_in_force
+                                               time_in_force=time_in_force
-                                       )
+                                               )
         except InsufficientFundsError as e:
             logger.warning(f"Unable to place order {e}.")
             # Try to figure out what went wrong
@@ -1113,7 +1142,7 @@ class FreqtradeBot(LoggingMixin):
         trade.close_rate_requested = limit
         trade.sell_reason = sell_reason.sell_reason
         # In case of market sell orders the order can be closed immediately
-        if order.get('status', 'unknown') == 'closed':
+        if order.get('status', 'unknown') in ('closed', 'expired'):
             self.update_trade_state(trade, trade.open_order_id, order)
         Trade.commit()
 
@@ -1352,7 +1381,9 @@ class FreqtradeBot(LoggingMixin):
         if fee_currency:
             # fee_rate should use mean
             fee_rate = sum(fee_rate_array) / float(len(fee_rate_array)) if fee_rate_array else None
-            trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', ''))
+            if fee_rate is not None and fee_rate < 0.02:
+                # Only update if fee-rate is < 2%
+                trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', ''))
 
         if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC):
             logger.warning(f"Amount {amount} does not match amount {trade.amount}")
@@ -1363,3 +1394,26 @@ class FreqtradeBot(LoggingMixin):
                                               amount=amount, fee_abs=fee_abs)
         else:
             return amount
+
+    def get_valid_price(self, custom_price: float, proposed_price: float) -> float:
+        """
+        Return the valid price.
+        Check if the custom price is of the good type if not return proposed_price
+        :return: valid price for the order
+        """
+        if custom_price:
+            try:
+                valid_custom_price = float(custom_price)
+            except ValueError:
+                valid_custom_price = proposed_price
+        else:
+            valid_custom_price = proposed_price
+
+        cust_p_max_dist_r = self.config.get('custom_price_max_distance_ratio', 0.02)
+        min_custom_price_allowed = proposed_price - (proposed_price * cust_p_max_dist_r)
+        max_custom_price_allowed = proposed_price + (proposed_price * cust_p_max_dist_r)
+
+        # Bracket between min_custom_price_allowed and max_custom_price_allowed
+        return max(
+            min(valid_custom_price, max_custom_price_allowed),
+            min_custom_price_allowed)

freqtrade/main.py

@@ -44,7 +44,7 @@ def main(sysargv: List[str] = None) -> None:
                 "as `freqtrade trade [options...]`.\n"
                 "To see the full list of options available, please use "
                 "`freqtrade --help` or `freqtrade <command> --help`."
-                )
+            )
 
     except SystemExit as e:
         return_code = e

freqtrade/optimize/backtesting.py

@@ -15,7 +15,7 @@ from freqtrade.configuration import TimeRange, remove_credentials, validate_conf
 from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data import history
 from freqtrade.data.btanalysis import trade_list_to_dataframe
-from freqtrade.data.converter import trim_dataframes
+from freqtrade.data.converter import trim_dataframe, trim_dataframes
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.enums import BacktestState, SellType
 from freqtrade.exceptions import DependencyException, OperationalException
@@ -43,6 +43,7 @@ CLOSE_IDX = 3
 SELL_IDX = 4
 LOW_IDX = 5
 HIGH_IDX = 6
+BUY_TAG_IDX = 7
 
 
 class Backtesting:
@@ -116,14 +117,22 @@ class Backtesting:
 
         self.wallets = Wallets(self.config, self.exchange, log=False)
 
+        self.timerange = TimeRange.parse_timerange(
+            None if self.config.get('timerange') is None else str(self.config.get('timerange')))
+
         # Get maximum required startup period
         self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
+        # Add maximum startup candle count to configuration for informative pairs support
+        self.config['startup_candle_count'] = self.required_startup
         self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
 
         self.progress = BTProgress()
         self.abort = False
 
     def __del__(self):
+        self.cleanup()
+
+    def cleanup(self):
         LoggingMixin.show_output = True
         PairLocks.use_db = True
         Trade.use_db = True
@@ -140,6 +149,8 @@ class Backtesting:
         # since a "perfect" stoploss-sell is assumed anyway
         # And the regular "stoploss" function would not apply to that case
         self.strategy.order_types['stoploss_on_exchange'] = False
+
+    def _load_protections(self, strategy: IStrategy):
         if self.config.get('enable_protections', False):
             conf = self.config
             if hasattr(strategy, 'protections'):
@@ -154,14 +165,11 @@ class Backtesting:
         """
         self.progress.init_step(BacktestState.DATALOAD, 1)
 
-        timerange = TimeRange.parse_timerange(None if self.config.get(
-            'timerange') is None else str(self.config.get('timerange')))
-
         data = history.load_data(
             datadir=self.config['datadir'],
             pairs=self.pairlists.whitelist,
             timeframe=self.timeframe,
-            timerange=timerange,
+            timerange=self.timerange,
             startup_candles=self.required_startup,
             fail_without_data=True,
             data_format=self.config.get('dataformat_ohlcv', 'json'),
@@ -174,11 +182,11 @@ class Backtesting:
                     f'({(max_date - min_date).days} days).')
 
         # Adjust startts forward if not enough data is available
-        timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
+        self.timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
-                                            self.required_startup, min_date)
+                                                 self.required_startup, min_date)
 
         self.progress.set_new_value(1)
-        return data, timerange
+        return data, self.timerange
 
     def prepare_backtest(self, enable_protections):
         """
@@ -191,6 +199,7 @@ class Backtesting:
         Trade.reset_trades()
         self.rejected_trades = 0
         self.dataprovider.clear_cache()
+        self._load_protections(self.strategy)
 
     def check_abort(self):
         """
@@ -209,7 +218,7 @@ class Backtesting:
         """
         # Every change to this headers list must evaluate further usages of the resulting tuple
         # and eventually change the constants for indexes at the top
-        headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
+        headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high', 'buy_tag']
         data: Dict = {}
         self.progress.init_step(BacktestState.CONVERT, len(processed))
 
@@ -220,20 +229,27 @@ class Backtesting:
             if not pair_data.empty:
                 pair_data.loc[:, 'buy'] = 0  # cleanup if buy_signal is exist
                 pair_data.loc[:, 'sell'] = 0  # cleanup if sell_signal is exist
+                pair_data.loc[:, 'buy_tag'] = None  # cleanup if buy_tag is exist
 
             df_analyzed = self.strategy.advise_sell(
-                self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
+                self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair}).copy()
-
+            # Trim startup period from analyzed dataframe
+            df_analyzed = trim_dataframe(df_analyzed, self.timerange,
+                                         startup_candles=self.required_startup)
             # To avoid using data from future, we use buy/sell signals shifted
             # from the previous candle
             df_analyzed.loc[:, 'buy'] = df_analyzed.loc[:, 'buy'].shift(1)
             df_analyzed.loc[:, 'sell'] = df_analyzed.loc[:, 'sell'].shift(1)
+            df_analyzed.loc[:, 'buy_tag'] = df_analyzed.loc[:, 'buy_tag'].shift(1)
 
-            df_analyzed.drop(df_analyzed.head(1).index, inplace=True)
+            # Update dataprovider cache
+            self.dataprovider._set_cached_df(pair, self.timeframe, df_analyzed)
+
+            df_analyzed = df_analyzed.drop(df_analyzed.head(1).index)
 
             # Convert from Pandas to list for performance reasons
             # (Looping Pandas is slow.)
-            data[pair] = df_analyzed.values.tolist()
+            data[pair] = df_analyzed[headers].values.tolist()
         return data
 
     def _get_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple,
@@ -262,7 +278,7 @@ class Backtesting:
                     # Worst case: price reaches stop_positive_offset and dives down.
                     stop_rate = (sell_row[OPEN_IDX] *
                                  (1 + abs(self.strategy.trailing_stop_positive_offset) -
-                                 abs(self.strategy.trailing_stop_positive)))
+                                  abs(self.strategy.trailing_stop_positive)))
                 else:
                     # Worst case: price ticks tiny bit above open and dives down.
                     stop_rate = sell_row[OPEN_IDX] * (1 - abs(trade.stop_loss_pct))
@@ -303,14 +319,14 @@ class Backtesting:
             return sell_row[OPEN_IDX]
 
     def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]:
-
+        sell_candle_time = sell_row[DATE_IDX].to_pydatetime()
         sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX],  # type: ignore
-                                         sell_row[DATE_IDX].to_pydatetime(), sell_row[BUY_IDX],
+                                         sell_candle_time, sell_row[BUY_IDX],
                                          sell_row[SELL_IDX],
                                          low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX])
 
         if sell.sell_flag:
-            trade.close_date = sell_row[DATE_IDX].to_pydatetime()
+            trade.close_date = sell_candle_time
             trade.sell_reason = sell.sell_reason
             trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60)
             closerate = self._get_close_rate(sell_row, trade, sell, trade_dur)
@@ -322,7 +338,7 @@ class Backtesting:
                     rate=closerate,
                     time_in_force=time_in_force,
                     sell_reason=sell.sell_reason,
-                    current_time=sell_row[DATE_IDX].to_pydatetime()):
+                    current_time=sell_candle_time):
                 return None
 
             trade.close(closerate, show_msg=False)
@@ -358,6 +374,7 @@ class Backtesting:
 
         if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount):
             # Enter trade
+            has_buy_tag = len(row) >= BUY_TAG_IDX + 1
             trade = LocalTrade(
                 pair=pair,
                 open_rate=row[OPEN_IDX],
@@ -367,6 +384,7 @@ class Backtesting:
                 fee_open=self.fee,
                 fee_close=self.fee,
                 is_open=True,
+                buy_tag=row[BUY_TAG_IDX] if has_buy_tag else None,
                 exchange='backtesting',
             )
             return trade
@@ -423,10 +441,6 @@ class Backtesting:
         trades: List[LocalTrade] = []
         self.prepare_backtest(enable_protections)
 
-        # Update dataprovider cache
-        for pair, dataframe in processed.items():
-            self.dataprovider._set_cached_df(pair, self.timeframe, dataframe)
-
         # Use dict of lists with data for performance
         # (looping lists is a lot faster than pandas DataFrames)
         data: Dict = self._get_ohlcv_as_lists(processed)
@@ -448,6 +462,8 @@ class Backtesting:
             for i, pair in enumerate(data):
                 row_index = indexes[pair]
                 try:
+                    # Row is treated as "current incomplete candle".
+                    # Buy / sell signals are shifted by 1 to compensate for this.
                     row = data[pair][row_index]
                 except IndexError:
                     # missing Data for one pair at the end.
@@ -459,8 +475,8 @@ class Backtesting:
                     continue
 
                 row_index += 1
-                self.dataprovider._set_dataframe_max_index(row_index)
                 indexes[pair] = row_index
+                self.dataprovider._set_dataframe_max_index(row_index)
 
                 # without positionstacking, we can only have one open trade per pair.
                 # max_open_trades must be respected
@@ -484,7 +500,7 @@ class Backtesting:
                         open_trades[pair].append(trade)
                         LocalTrade.add_bt_trade(trade)
 
-                for trade in open_trades[pair]:
+                for trade in list(open_trades[pair]):
                     # also check the buying candle for sell conditions.
                     trade_entry = self._get_sell_trade_entry(trade, row)
                     # Sell occurred
@@ -515,7 +531,8 @@ class Backtesting:
             'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']),
         }
 
-    def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange):
+    def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, DataFrame],
+                              timerange: TimeRange):
         self.progress.init_step(BacktestState.ANALYZE, 0)
 
         logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
@@ -534,17 +551,18 @@ class Backtesting:
             max_open_trades = 0
 
         # need to reprocess data every time to populate signals
-        preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
+        preprocessed = self.strategy.advise_all_indicators(data)
 
         # Trim startup period from analyzed dataframe
-        preprocessed = trim_dataframes(preprocessed, timerange, self.required_startup)
+        preprocessed_tmp = trim_dataframes(preprocessed, timerange, self.required_startup)
 
-        if not preprocessed:
+        if not preprocessed_tmp:
             raise OperationalException(
                 "No data left after adjusting for startup candles.")
 
-        min_date, max_date = history.get_timerange(preprocessed)
+        # Use preprocessed_tmp for date generation (the trimmed dataframe).
-
+        # Backtesting will re-trim the dataframes after buy/sell signal generation.
+        min_date, max_date = history.get_timerange(preprocessed_tmp)
         logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'({(max_date - min_date).days} days).')

freqtrade/optimize/hyperopt.py

@@ -66,6 +66,7 @@ class Hyperopt:
     def __init__(self, config: Dict[str, Any]) -> None:
         self.buy_space: List[Dimension] = []
         self.sell_space: List[Dimension] = []
+        self.protection_space: List[Dimension] = []
         self.roi_space: List[Dimension] = []
         self.stoploss_space: List[Dimension] = []
         self.trailing_space: List[Dimension] = []
@@ -102,16 +103,30 @@ class Hyperopt:
         self.num_epochs_saved = 0
         self.current_best_epoch: Optional[Dict[str, Any]] = None
 
-        # Populate functions here (hasattr is slow so should not be run during "regular" operations)
+        if not self.auto_hyperopt:
-        if hasattr(self.custom_hyperopt, 'populate_indicators'):
+            # Populate "fallback" functions here
-            self.backtesting.strategy.advise_indicators = (  # type: ignore
+            # (hasattr is slow so should not be run during "regular" operations)
-                self.custom_hyperopt.populate_indicators)  # type: ignore
+            if hasattr(self.custom_hyperopt, 'populate_indicators'):
-        if hasattr(self.custom_hyperopt, 'populate_buy_trend'):
+                logger.warning(
-            self.backtesting.strategy.advise_buy = (  # type: ignore
+                    "DEPRECATED: Using `populate_indicators()` in the hyperopt file is deprecated. "
-                self.custom_hyperopt.populate_buy_trend)  # type: ignore
+                    "Please move these methods to your strategy."
-        if hasattr(self.custom_hyperopt, 'populate_sell_trend'):
+                    )
-            self.backtesting.strategy.advise_sell = (  # type: ignore
+                self.backtesting.strategy.populate_indicators = (  # type: ignore
-                self.custom_hyperopt.populate_sell_trend)  # type: ignore
+                    self.custom_hyperopt.populate_indicators)  # type: ignore
+            if hasattr(self.custom_hyperopt, 'populate_buy_trend'):
+                logger.warning(
+                    "DEPRECATED: Using `populate_buy_trend()` in the hyperopt file is deprecated. "
+                    "Please move these methods to your strategy."
+                )
+                self.backtesting.strategy.populate_buy_trend = (  # type: ignore
+                    self.custom_hyperopt.populate_buy_trend)  # type: ignore
+            if hasattr(self.custom_hyperopt, 'populate_sell_trend'):
+                logger.warning(
+                    "DEPRECATED: Using `populate_sell_trend()` in the hyperopt file is deprecated. "
+                    "Please move these methods to your strategy."
+                )
+                self.backtesting.strategy.populate_sell_trend = (  # type: ignore
+                    self.custom_hyperopt.populate_sell_trend)  # type: ignore
 
         # Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set
         if self.config.get('use_max_market_positions', True):
@@ -189,6 +204,8 @@ class Hyperopt:
             result['buy'] = {p.name: params.get(p.name) for p in self.buy_space}
         if HyperoptTools.has_space(self.config, 'sell'):
             result['sell'] = {p.name: params.get(p.name) for p in self.sell_space}
+        if HyperoptTools.has_space(self.config, 'protection'):
+            result['protection'] = {p.name: params.get(p.name) for p in self.protection_space}
         if HyperoptTools.has_space(self.config, 'roi'):
             result['roi'] = {str(k): v for k, v in
                              self.custom_hyperopt.generate_roi_table(params).items()}
@@ -239,6 +256,12 @@ class Hyperopt:
         """
         Assign the dimensions in the hyperoptimization space.
         """
+        if self.auto_hyperopt and HyperoptTools.has_space(self.config, 'protection'):
+            # Protections can only be optimized when using the Parameter interface
+            logger.debug("Hyperopt has 'protection' space")
+            # Enable Protections if protection space is selected.
+            self.config['enable_protections'] = True
+            self.protection_space = self.custom_hyperopt.protection_space()
 
         if HyperoptTools.has_space(self.config, 'buy'):
             logger.debug("Hyperopt has 'buy' space")
@@ -259,22 +282,19 @@ class Hyperopt:
         if HyperoptTools.has_space(self.config, 'trailing'):
             logger.debug("Hyperopt has 'trailing' space")
             self.trailing_space = self.custom_hyperopt.trailing_space()
-        self.dimensions = (self.buy_space + self.sell_space + self.roi_space +
+        self.dimensions = (self.buy_space + self.sell_space + self.protection_space
-                           self.stoploss_space + self.trailing_space)
+                           + self.roi_space + self.stoploss_space + self.trailing_space)
 
     def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict:
         """
-        Used Optimize function. Called once per epoch to optimize whatever is configured.
+        Used Optimize function.
+        Called once per epoch to optimize whatever is configured.
         Keep this function as optimized as possible!
         """
         backtest_start_time = datetime.now(timezone.utc)
         params_dict = self._get_params_dict(self.dimensions, raw_params)
 
         # Apply parameters
-        if HyperoptTools.has_space(self.config, 'roi'):
-            self.backtesting.strategy.minimal_roi = (  # type: ignore
-                self.custom_hyperopt.generate_roi_table(params_dict))
-
         if HyperoptTools.has_space(self.config, 'buy'):
             self.backtesting.strategy.advise_buy = (  # type: ignore
                 self.custom_hyperopt.buy_strategy_generator(params_dict))
@@ -283,6 +303,16 @@ class Hyperopt:
             self.backtesting.strategy.advise_sell = (  # type: ignore
                 self.custom_hyperopt.sell_strategy_generator(params_dict))
 
+        if HyperoptTools.has_space(self.config, 'protection'):
+            for attr_name, attr in self.backtesting.strategy.enumerate_parameters('protection'):
+                if attr.optimize:
+                    # noinspection PyProtectedMember
+                    attr.value = params_dict[attr_name]
+
+        if HyperoptTools.has_space(self.config, 'roi'):
+            self.backtesting.strategy.minimal_roi = (  # type: ignore
+                self.custom_hyperopt.generate_roi_table(params_dict))
+
         if HyperoptTools.has_space(self.config, 'stoploss'):
             self.backtesting.strategy.stoploss = params_dict['stoploss']
 
@@ -376,18 +406,17 @@ class Hyperopt:
         data, timerange = self.backtesting.load_bt_data()
         logger.info("Dataload complete. Calculating indicators")
 
-        preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
+        preprocessed = self.backtesting.strategy.advise_all_indicators(data)
 
-        # Trim startup period from analyzed dataframe
+        # Trim startup period from analyzed dataframe to get correct dates for output.
         processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup)
-
         self.min_date, self.max_date = get_timerange(processed)
 
         logger.info(f'Hyperopting with data from {self.min_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'({(self.max_date - self.min_date).days} days)..')
-
+        # Store non-trimmed data - will be trimmed after signal generation.
-        dump(processed, self.data_pickle_file)
+        dump(preprocessed, self.data_pickle_file)
 
     def start(self) -> None:
         self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
@@ -442,9 +471,9 @@ class Hyperopt:
                         ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
                     ]
                 with progressbar.ProgressBar(
-                         max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
+                    max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
-                         widgets=widgets
+                    widgets=widgets
-                     ) as pbar:
+                ) as pbar:
                     EVALS = ceil(self.total_epochs / jobs)
                     for i in range(EVALS):
                         # Correct the number of epochs to be processed for the last

freqtrade/optimize/hyperopt_auto.py

@@ -73,6 +73,9 @@ class HyperOptAuto(IHyperOpt):
     def sell_indicator_space(self) -> List['Dimension']:
         return self._get_indicator_space('sell', 'sell_indicator_space')
 
+    def protection_space(self) -> List['Dimension']:
+        return self._get_indicator_space('protection', 'protection_space')
+
     def generate_roi_table(self, params: Dict) -> Dict[int, float]:
         return self._get_func('generate_roi_table')(params)
 

freqtrade/optimize/hyperopt_epoch_filters.py

@@ -0,0 +1,128 @@
+import logging
+from typing import List
+
+from freqtrade.exceptions import OperationalException
+
+
+logger = logging.getLogger(__name__)
+
+
+def hyperopt_filter_epochs(epochs: List, filteroptions: dict, log: bool = True) -> List:
+    """
+    Filter our items from the list of hyperopt results
+    """
+    if filteroptions['only_best']:
+        epochs = [x for x in epochs if x['is_best']]
+    if filteroptions['only_profitable']:
+        epochs = [x for x in epochs
+                  if x['results_metrics'].get('profit_total', 0) > 0]
+
+    epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions)
+    if log:
+        logger.info(f"{len(epochs)} " +
+                    ("best " if filteroptions['only_best'] else "") +
+                    ("profitable " if filteroptions['only_profitable'] else "") +
+                    "epochs found.")
+    return epochs
+
+
+def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int):
+    """
+    Filter epochs with trade-counts > trades
+    """
+    return [
+        x for x in epochs if x['results_metrics'].get('total_trades', 0) > trade_count
+    ]
+
+
+def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
+
+    if filteroptions['filter_min_trades'] > 0:
+        epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades'])
+
+    if filteroptions['filter_max_trades'] > 0:
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get('total_trades') < filteroptions['filter_max_trades']
+        ]
+    return epochs
+
+
+def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
+
+    def get_duration_value(x):
+        # Duration in minutes ...
+        if 'holding_avg_s' in x['results_metrics']:
+            avg = x['results_metrics']['holding_avg_s']
+            return avg // 60
+        raise OperationalException(
+            "Holding-average not available. Please omit the filter on average time, "
+            "or rerun hyperopt with this version")
+
+    if filteroptions['filter_min_avg_time'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if get_duration_value(x) > filteroptions['filter_min_avg_time']
+        ]
+    if filteroptions['filter_max_avg_time'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if get_duration_value(x) < filteroptions['filter_max_avg_time']
+        ]
+
+    return epochs
+
+
+def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
+
+    if filteroptions['filter_min_avg_profit'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get('profit_mean', 0) * 100
+            > filteroptions['filter_min_avg_profit']
+        ]
+    if filteroptions['filter_max_avg_profit'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get('profit_mean', 0) * 100
+            < filteroptions['filter_max_avg_profit']
+        ]
+    if filteroptions['filter_min_total_profit'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get('profit_total_abs', 0)
+            > filteroptions['filter_min_total_profit']
+        ]
+    if filteroptions['filter_max_total_profit'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get('profit_total_abs', 0)
+            < filteroptions['filter_max_total_profit']
+        ]
+    return epochs
+
+
+def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
+
+    if filteroptions['filter_min_objective'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+
+        epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
+    if filteroptions['filter_max_objective'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+
+        epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
+
+    return epochs

freqtrade/optimize/hyperopt_interface.py

@@ -57,6 +57,13 @@ class IHyperOpt(ABC):
         """
         raise OperationalException(_format_exception_message('sell_strategy_generator', 'sell'))
 
+    def protection_space(self) -> List[Dimension]:
+        """
+        Create a protection space.
+        Only supported by the Parameter interface.
+        """
+        raise OperationalException(_format_exception_message('indicator_space', 'protection'))
+
     def indicator_space(self) -> List[Dimension]:
         """
         Create an indicator space.

freqtrade/optimize/hyperopt_tools.py

@@ -4,7 +4,7 @@ import logging
 from copy import deepcopy
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Iterator, List, Optional, Tuple
 
 import numpy as np
 import rapidjson
@@ -15,6 +15,7 @@ from pandas import isna, json_normalize
 from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2
+from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs
 
 
 logger = logging.getLogger(__name__)
@@ -82,53 +83,77 @@ class HyperoptTools():
         """
         Tell if the space value is contained in the configuration
         """
-        # The 'trailing' space is not included in the 'default' set of spaces
+        # 'trailing' and 'protection spaces are not included in the 'default' set of spaces
-        if space == 'trailing':
+        if space in ('trailing', 'protection'):
             return any(s in config['spaces'] for s in [space, 'all'])
         else:
             return any(s in config['spaces'] for s in [space, 'all', 'default'])
 
     @staticmethod
-    def _read_results_pickle(results_file: Path) -> List:
+    def _read_results(results_file: Path, batch_size: int = 10) -> Iterator[List[Any]]:
         """
-        Read hyperopt results from pickle file
+        Stream hyperopt results from file
-        LEGACY method - new files are written as json and cannot be read with this method.
-        """
-        from joblib import load
-
-        logger.info(f"Reading pickled epochs from '{results_file}'")
-        data = load(results_file)
-        return data
-
-    @staticmethod
-    def _read_results(results_file: Path) -> List:
-        """
-        Read hyperopt results from file
         """
         import rapidjson
         logger.info(f"Reading epochs from '{results_file}'")
         with results_file.open('r') as f:
-            data = [rapidjson.loads(line) for line in f]
+            data = []
-        return data
+            for line in f:
+                data += [rapidjson.loads(line)]
+                if len(data) >= batch_size:
+                    yield data
+                    data = []
+        yield data
 
     @staticmethod
-    def load_previous_results(results_file: Path) -> List:
+    def _test_hyperopt_results_exist(results_file) -> bool:
-        """
-        Load data for epochs from the file if we have one
-        """
-        epochs: List = []
         if results_file.is_file() and results_file.stat().st_size > 0:
             if results_file.suffix == '.pickle':
-                epochs = HyperoptTools._read_results_pickle(results_file)
+                raise OperationalException(
-            else:
+                    "Legacy hyperopt results are no longer supported."
-                epochs = HyperoptTools._read_results(results_file)
+                    "Please rerun hyperopt or use an older version to load this file."
-            # Detection of some old format, without 'is_best' field saved
+                )
-            if epochs[0].get('is_best') is None:
+            return True
+        else:
+            # No file found.
+            return False
+
+    @staticmethod
+    def load_filtered_results(results_file: Path, config: Dict[str, Any]) -> Tuple[List, int]:
+        filteroptions = {
+            'only_best': config.get('hyperopt_list_best', False),
+            'only_profitable': config.get('hyperopt_list_profitable', False),
+            'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
+            'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
+            'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
+            'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
+            'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
+            'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
+            'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
+            'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
+            'filter_min_objective': config.get('hyperopt_list_min_objective', None),
+            'filter_max_objective': config.get('hyperopt_list_max_objective', None),
+        }
+        if not HyperoptTools._test_hyperopt_results_exist(results_file):
+            # No file found.
+            return [], 0
+
+        epochs = []
+        total_epochs = 0
+        for epochs_tmp in HyperoptTools._read_results(results_file):
+            if total_epochs == 0 and epochs_tmp[0].get('is_best') is None:
                 raise OperationalException(
                     "The file with HyperoptTools results is incompatible with this version "
                     "of Freqtrade and cannot be loaded.")
-            logger.info(f"Loaded {len(epochs)} previous evaluations from disk.")
+            total_epochs += len(epochs_tmp)
-        return epochs
+            epochs += hyperopt_filter_epochs(epochs_tmp, filteroptions, log=False)
+
+        logger.info(f"Loaded {total_epochs} previous evaluations from disk.")
+
+        # Final filter run ...
+        epochs = hyperopt_filter_epochs(epochs, filteroptions, log=True)
+
+        return epochs, total_epochs
 
     @staticmethod
     def show_epoch_details(results, total_epochs: int, print_json: bool,
@@ -149,7 +174,7 @@ class HyperoptTools():
 
         if print_json:
             result_dict: Dict = {}
-            for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']:
+            for s in ['buy', 'sell', 'protection', 'roi', 'stoploss', 'trailing']:
                 HyperoptTools._params_update_for_json(result_dict, params, non_optimized, s)
             print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE))
 
@@ -158,6 +183,8 @@ class HyperoptTools():
                                                non_optimized)
             HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:",
                                                non_optimized)
+            HyperoptTools._params_pretty_print(params, 'protection',
+                                               "Protection hyperspace params:", non_optimized)
             HyperoptTools._params_pretty_print(params, 'roi', "ROI table:", non_optimized)
             HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:", non_optimized)
             HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:", non_optimized)
@@ -203,7 +230,7 @@ class HyperoptTools():
             elif space == "roi":
                 result = result[:-1] + f'{appendix}\n'
                 minimal_roi_result = rapidjson.dumps({
-                        str(k): v for k, v in (space_params or no_params).items()
+                    str(k): v for k, v in (space_params or no_params).items()
                 }, default=str, indent=4, number_mode=rapidjson.NM_NATIVE)
                 result += f"minimal_roi = {minimal_roi_result}"
             elif space == "trailing":
@@ -431,21 +458,14 @@ class HyperoptTools():
         trials['Best'] = ''
         trials['Stake currency'] = config['stake_currency']
 
-        if 'results_metrics.total_trades' in trials:
+        base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades',
-            base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades',
+                        'results_metrics.profit_mean', 'results_metrics.profit_median',
-                            'results_metrics.profit_mean', 'results_metrics.profit_median',
+                        'results_metrics.profit_total',
-                            'results_metrics.profit_total',
+                        'Stake currency',
-                            'Stake currency',
+                        'results_metrics.profit_total_abs', 'results_metrics.holding_avg',
-                            'results_metrics.profit_total_abs', 'results_metrics.holding_avg',
+                        'loss', 'is_initial_point', 'is_best']
-                            'loss', 'is_initial_point', 'is_best']
+        perc_multi = 100
-            perc_multi = 100
+
-        else:
-            perc_multi = 1
-            base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count',
-                            'results_metrics.avg_profit', 'results_metrics.median_profit',
-                            'results_metrics.total_profit',
-                            'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
-                            'loss', 'is_initial_point', 'is_best']
         param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()]
         trials = trials[base_metrics + param_metrics]
 
@@ -473,11 +493,6 @@ class HyperoptTools():
         trials['Avg profit'] = trials['Avg profit'].apply(
             lambda x: f'{x * perc_multi:,.2f}%' if not isna(x) else ""
         )
-        if perc_multi == 1:
-            trials['Avg duration'] = trials['Avg duration'].apply(
-                lambda x: f'{x:,.1f} m' if isinstance(
-                    x, float) else f"{x.total_seconds() // 60:,.1f} m" if not isna(x) else ""
-            )
         trials['Objective'] = trials['Objective'].apply(
             lambda x: f'{x:,.5f}' if x != 100000 else ""
         )

freqtrade/optimize/optimize_reports.py

@@ -31,7 +31,7 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N
         filename = Path.joinpath(
             recordfilename.parent,
             f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}'
-            ).with_suffix(recordfilename.suffix)
+        ).with_suffix(recordfilename.suffix)
     file_dump_json(filename, stats)
 
     latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN)
@@ -173,7 +173,7 @@ def generate_strategy_comparison(all_results: Dict) -> List[Dict]:
     for strategy, results in all_results.items():
         tabular_data.append(_generate_result_line(
             results['results'], results['config']['dry_run_wallet'], strategy)
-            )
+        )
         try:
             max_drawdown_per, _, _, _, _ = calculate_max_drawdown(results['results'],
                                                                   value_col='profit_ratio')
@@ -604,7 +604,7 @@ def text_table_add_metrics(strat_results: Dict) -> str:
                                          strat_results['stake_currency'])
         stake_amount = round_coin_value(
             strat_results['stake_amount'], strat_results['stake_currency']
-            ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited'
+        ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited'
 
         message = ("No trades made. "
                    f"Your starting balance was {start_balance}, "

freqtrade/persistence/migrations.py

@@ -47,6 +47,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
     min_rate = get_column_def(cols, 'min_rate', 'null')
     sell_reason = get_column_def(cols, 'sell_reason', 'null')
     strategy = get_column_def(cols, 'strategy', 'null')
+    buy_tag = get_column_def(cols, 'buy_tag', 'null')
     # If ticker-interval existed use that, else null.
     if has_column(cols, 'ticker_interval'):
         timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
@@ -64,7 +65,8 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
     # Schema migration necessary
     with engine.begin() as connection:
         connection.execute(text(f"alter table trades rename to {table_back_name}"))
-        # drop indexes on backup table
+    with engine.begin() as connection:
+        # drop indexes on backup table in new session
         for index in inspector.get_indexes(table_back_name):
             connection.execute(text(f"drop index {index['name']}"))
     # let SQLAlchemy create the schema as required
@@ -75,22 +77,15 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
         connection.execute(text(f"""insert into trades
             (id, exchange, pair, is_open,
             fee_open, fee_open_cost, fee_open_currency,
-            fee_close, fee_close_cost, fee_open_currency, open_rate,
+            fee_close, fee_close_cost, fee_close_currency, open_rate,
             open_rate_requested, close_rate, close_rate_requested, close_profit,
             stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
             stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
             stoploss_order_id, stoploss_last_update,
-            max_rate, min_rate, sell_reason, sell_order_status, strategy,
+            max_rate, min_rate, sell_reason, sell_order_status, strategy, buy_tag,
             timeframe, open_trade_value, close_profit_abs
             )
-        select id, lower(exchange),
+        select id, lower(exchange), pair,
-            case
-                when instr(pair, '_') != 0 then
-                substr(pair,    instr(pair, '_') + 1) || '/' ||
-                substr(pair, 1, instr(pair, '_') - 1)
-                else pair
-                end
-            pair,
             is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
             {fee_open_currency} fee_open_currency, {fee_close} fee_close,
             {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
@@ -103,7 +98,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
             {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
             {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
             {sell_order_status} sell_order_status,
-            {strategy} strategy, {timeframe} timeframe,
+            {strategy} strategy, {buy_tag} buy_tag, {timeframe} timeframe,
             {open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs
             from {table_back_name}
             """))
@@ -131,7 +126,9 @@ def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, col
 
     with engine.begin() as connection:
         connection.execute(text(f"alter table orders rename to {table_back_name}"))
-        # drop indexes on backup table
+
+    with engine.begin() as connection:
+        # drop indexes on backup table in new session
         for index in inspector.get_indexes(table_back_name):
             connection.execute(text(f"drop index {index['name']}"))
 
@@ -160,7 +157,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
     table_back_name = get_backup_name(tabs, 'trades_bak')
 
     # Check for latest column
-    if not has_column(cols, 'open_trade_value'):
+    if not has_column(cols, 'buy_tag'):
         logger.info(f'Running database migration for trades - backup: {table_back_name}')
         migrate_trades_table(decl_base, inspector, engine, table_back_name, cols)
         # Reread columns - the above recreated the table!

freqtrade/persistence/models.py

@@ -13,7 +13,7 @@ from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session
 from sqlalchemy.pool import StaticPool
 from sqlalchemy.sql.schema import UniqueConstraint
 
-from freqtrade.constants import DATETIME_PRINT_FORMAT
+from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES
 from freqtrade.enums import SellType
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.misc import safe_value_fallback
@@ -159,9 +159,9 @@ class Order(_DECL_BASE):
             self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc)
 
         self.ft_is_open = True
-        if self.status in ('closed', 'canceled', 'cancelled'):
+        if self.status in NON_OPEN_EXCHANGE_STATES:
             self.ft_is_open = False
-            if order.get('filled', 0) > 0:
+            if (order.get('filled', 0.0) or 0.0) > 0:
                 self.order_filled_date = datetime.now(timezone.utc)
         self.order_update_date = datetime.now(timezone.utc)
 
@@ -257,6 +257,7 @@ class LocalTrade():
     sell_reason: str = ''
     sell_order_status: str = ''
     strategy: str = ''
+    buy_tag: Optional[str] = None
     timeframe: Optional[int] = None
 
     def __init__(self, **kwargs):
@@ -288,6 +289,7 @@ class LocalTrade():
             'amount_requested': round(self.amount_requested, 8) if self.amount_requested else None,
             'stake_amount': round(self.stake_amount, 8),
             'strategy': self.strategy,
+            'buy_tag': self.buy_tag,
             'timeframe': self.timeframe,
 
             'fee_open': self.fee_open,
@@ -352,12 +354,12 @@ class LocalTrade():
         LocalTrade.trades_open = []
         LocalTrade.total_profit = 0
 
-    def adjust_min_max_rates(self, current_price: float) -> None:
+    def adjust_min_max_rates(self, current_price: float, current_price_low: float) -> None:
         """
         Adjust the max_rate and min_rate.
         """
         self.max_rate = max(current_price, self.max_rate or self.open_rate)
-        self.min_rate = min(current_price, self.min_rate or self.open_rate)
+        self.min_rate = min(current_price_low, self.min_rate or self.open_rate)
 
     def _set_new_stoploss(self, new_loss: float, stoploss: float):
         """Assign new stop value"""
@@ -636,7 +638,7 @@ class LocalTrade():
 
             # skip case if trailing-stop changed the stoploss already.
             if (trade.stop_loss == trade.initial_stop_loss
-               and trade.initial_stop_loss_pct != desired_stoploss):
+                    and trade.initial_stop_loss_pct != desired_stoploss):
                 # Stoploss value got changed
 
                 logger.info(f"Stoploss for {trade} needs adjustment...")
@@ -703,6 +705,7 @@ class Trade(_DECL_BASE, LocalTrade):
     sell_reason = Column(String(100), nullable=True)
     sell_order_status = Column(String(100), nullable=True)
     strategy = Column(String(100), nullable=True)
+    buy_tag = Column(String(100), nullable=True)
     timeframe = Column(Integer, nullable=True)
 
     def __init__(self, **kwargs):

freqtrade/plot/plotting.py

@@ -334,8 +334,8 @@ def add_areas(fig, row: int, data: pd.DataFrame, indicators) -> make_subplots:
                 )
             elif indicator_b not in data:
                 logger.info(
-                        'fill_to: "%s" ignored. Reason: This indicator is not '
+                    'fill_to: "%s" ignored. Reason: This indicator is not '
-                        'in your strategy.', indicator_b
+                    'in your strategy.', indicator_b
                 )
     return fig
 
@@ -538,7 +538,7 @@ def load_and_plot_trades(config: Dict[str, Any]):
     - Initializes plot-script
     - Get candle (OHLCV) data
     - Generate Dafaframes populated with indicators and signals based on configured strategy
-    - Load trades excecuted during the selected period
+    - Load trades executed during the selected period
     - Generate Plotly plot objects
     - Generate plot files
     :return: None

freqtrade/plugins/pairlist/IPairList.py

@@ -144,24 +144,26 @@ class IPairList(LoggingMixin, ABC):
         markets = self._exchange.markets
         if not markets:
             raise OperationalException(
-                    'Markets not loaded. Make sure that exchange is initialized correctly.')
+                'Markets not loaded. Make sure that exchange is initialized correctly.')
 
         sanitized_whitelist: List[str] = []
         for pair in pairlist:
             # pair is not in the generated dynamic market or has the wrong stake currency
             if pair not in markets:
-                logger.warning(f"Pair {pair} is not compatible with exchange "
+                self.log_once(f"Pair {pair} is not compatible with exchange "
-                               f"{self._exchange.name}. Removing it from whitelist..")
+                              f"{self._exchange.name}. Removing it from whitelist..",
+                              logger.warning)
                 continue
 
             if not self._exchange.market_is_tradable(markets[pair]):
-                logger.warning(f"Pair {pair} is not tradable with Freqtrade."
+                self.log_once(f"Pair {pair} is not tradable with Freqtrade."
-                               "Removing it from whitelist..")
+                              "Removing it from whitelist..", logger.warning)
                 continue
 
             if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']:
-                logger.warning(f"Pair {pair} is not compatible with your stake currency "
+                self.log_once(f"Pair {pair} is not compatible with your stake currency "
-                               f"{self._config['stake_currency']}. Removing it from whitelist..")
+                              f"{self._config['stake_currency']}. Removing it from whitelist..",
+                              logger.warning)
                 continue
 
             # Check if market is active

freqtrade/plugins/pairlist/VolumePairList.py

@@ -4,6 +4,7 @@ Volume PairList provider
 Provides dynamic pair list based on trade volumes
 """
 import logging
+from functools import partial
 from typing import Any, Dict, List
 
 import arrow
@@ -115,18 +116,18 @@ class VolumePairList(IPairList):
         pairlist = self._pair_cache.get('pairlist')
         if pairlist:
             # Item found - no refresh necessary
-            return pairlist
+            return pairlist.copy()
         else:
             # Use fresh pairlist
             # Check if pair quote currency equals to the stake currency.
             filtered_tickers = [
-                    v for k, v in tickers.items()
+                v for k, v in tickers.items()
-                    if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
+                if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
-                        and v[self._sort_key] is not None)]
+                    and v[self._sort_key] is not None)]
             pairlist = [s['symbol'] for s in filtered_tickers]
 
             pairlist = self.filter_pairlist(pairlist, tickers)
-            self._pair_cache['pairlist'] = pairlist
+            self._pair_cache['pairlist'] = pairlist.copy()
 
         return pairlist
 
@@ -197,13 +198,13 @@ class VolumePairList(IPairList):
 
         if self._min_value > 0:
             filtered_tickers = [
-                    v for v in filtered_tickers if v[self._sort_key] > self._min_value]
+                v for v in filtered_tickers if v[self._sort_key] > self._min_value]
 
         sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key])
 
         # Validate whitelist to only have active market pairs
         pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
-        pairs = self.verify_blacklist(pairs, logger.info)
+        pairs = self.verify_blacklist(pairs, partial(self.log_once, logmethod=logger.info))
         # Limit pairlist to the requested number of pairs
         pairs = pairs[:self._number_pairs]
 

freqtrade/plugins/pairlist/rangestabilityfilter.py

@@ -26,6 +26,7 @@ class RangeStabilityFilter(IPairList):
 
         self._days = pairlistconfig.get('lookback_days', 10)
         self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01)
+        self._max_rate_of_change = pairlistconfig.get('max_rate_of_change', None)
         self._refresh_period = pairlistconfig.get('refresh_period', 1440)
 
         self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
@@ -50,8 +51,12 @@ class RangeStabilityFilter(IPairList):
         """
         Short whitelist method description - used for startup-messages
         """
+        max_rate_desc = ""
+        if self._max_rate_of_change:
+            max_rate_desc = (f" and above {self._max_rate_of_change}")
         return (f"{self.name} - Filtering pairs with rate of change below "
-                f"{self._min_rate_of_change} over the last {plural(self._days, 'day')}.")
+                f"{self._min_rate_of_change}{max_rate_desc} over the "
+                f"last {plural(self._days, 'day')}.")
 
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
@@ -104,6 +109,17 @@ class RangeStabilityFilter(IPairList):
                               f"which is below the threshold of {self._min_rate_of_change}.",
                               logger.info)
                 result = False
+            if self._max_rate_of_change:
+                if pct_change <= self._max_rate_of_change:
+                    result = True
+                else:
+                    self.log_once(
+                        f"Removed {pair} from whitelist, because rate of change "
+                        f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
+                        f"which is above the threshold of {self._max_rate_of_change}.",
+                        logger.info)
+                    result = False
             self._pair_cache[pair] = result
-
+        else:
+            self.log_once(f"Removed {pair} from whitelist, no candles found.", logger.info)
         return result

freqtrade/plugins/pairlistmanager.py

@@ -28,13 +28,13 @@ class PairListManager():
         self._tickers_needed = False
         for pairlist_handler_config in self._config.get('pairlists', None):
             pairlist_handler = PairListResolver.load_pairlist(
-                    pairlist_handler_config['method'],
+                pairlist_handler_config['method'],
-                    exchange=exchange,
+                exchange=exchange,
-                    pairlistmanager=self,
+                pairlistmanager=self,
-                    config=config,
+                config=config,
-                    pairlistconfig=pairlist_handler_config,
+                pairlistconfig=pairlist_handler_config,
-                    pairlist_pos=len(self._pairlist_handlers)
+                pairlist_pos=len(self._pairlist_handlers)
-                    )
+            )
             self._tickers_needed |= pairlist_handler.needstickers
             self._pairlist_handlers.append(pairlist_handler)
 

freqtrade/plugins/protections/iprotection.py

@@ -25,19 +25,22 @@ class IProtection(LoggingMixin, ABC):
     def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None:
         self._config = config
         self._protection_config = protection_config
+        self._stop_duration_candles: Optional[int] = None
+        self._lookback_period_candles: Optional[int] = None
+
         tf_in_min = timeframe_to_minutes(config['timeframe'])
         if 'stop_duration_candles' in protection_config:
-            self._stop_duration_candles = protection_config.get('stop_duration_candles', 1)
+            self._stop_duration_candles = int(protection_config.get('stop_duration_candles', 1))
             self._stop_duration = (tf_in_min * self._stop_duration_candles)
         else:
             self._stop_duration_candles = None
             self._stop_duration = protection_config.get('stop_duration', 60)
         if 'lookback_period_candles' in protection_config:
-            self._lookback_period_candles = protection_config.get('lookback_period_candles', 1)
+            self._lookback_period_candles = int(protection_config.get('lookback_period_candles', 1))
             self._lookback_period = tf_in_min * self._lookback_period_candles
         else:
             self._lookback_period_candles = None
-            self._lookback_period = protection_config.get('lookback_period', 60)
+            self._lookback_period = int(protection_config.get('lookback_period', 60))
 
         LoggingMixin.__init__(self, logger)
 

freqtrade/plugins/protections/stoploss_guard.py

@@ -54,9 +54,9 @@ class StoplossGuard(IProtection):
 
         trades1 = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until)
         trades = [trade for trade in trades1 if (str(trade.sell_reason) in (
-                    SellType.TRAILING_STOP_LOSS.value, SellType.STOP_LOSS.value,
+            SellType.TRAILING_STOP_LOSS.value, SellType.STOP_LOSS.value,
-                    SellType.STOPLOSS_ON_EXCHANGE.value)
+            SellType.STOPLOSS_ON_EXCHANGE.value)
-                      and trade.close_profit and trade.close_profit < 0)]
+            and trade.close_profit and trade.close_profit < 0)]
 
         if len(trades) < self._trade_limit:
             return False, None, None

freqtrade/resolvers/__init__.py

@@ -8,6 +8,3 @@ from freqtrade.resolvers.exchange_resolver import ExchangeResolver
 from freqtrade.resolvers.pairlist_resolver import PairListResolver
 from freqtrade.resolvers.protection_resolver import ProtectionResolver
 from freqtrade.resolvers.strategy_resolver import StrategyResolver
-
-
-

freqtrade/resolvers/strategy_resolver.py

@@ -50,7 +50,7 @@ class StrategyResolver(IResolver):
             if 'timeframe' not in config:
                 logger.warning(
                     "DEPRECATED: Please migrate to using 'timeframe' instead of 'ticker_interval'."
-                    )
+                )
                 strategy.timeframe = strategy.ticker_interval
 
         if strategy._ft_params_from_file:
@@ -119,7 +119,7 @@ class StrategyResolver(IResolver):
         - default (if not None)
         """
         if (attribute in config
-                and not isinstance(getattr(type(strategy), 'my_property', None), property)):
+                and not isinstance(getattr(type(strategy), attribute, None), property)):
             # Ensure Properties are not overwritten
             setattr(strategy, attribute, config[attribute])
             logger.info("Override strategy '%s' with value in config file: %s.",

freqtrade/rpc/api_server/api_backtest.py

@@ -47,15 +47,15 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
                 not ApiServer._bt
                 or lastconfig.get('timeframe') != strat.timeframe
                 or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0)
+                or lastconfig.get('timerange') != btconfig['timerange']
             ):
                 from freqtrade.optimize.backtesting import Backtesting
                 ApiServer._bt = Backtesting(btconfig)
 
-            # Only reload data if timeframe or timerange changed.
+            # Only reload data if timeframe changed.
             if (
                 not ApiServer._bt_data
                 or not ApiServer._bt_timerange
-                or lastconfig.get('timerange') != btconfig['timerange']
                 or lastconfig.get('stake_amount') != btconfig.get('stake_amount')
                 or lastconfig.get('enable_protections') != btconfig.get('enable_protections')
                 or lastconfig.get('protections') != btconfig.get('protections', [])

freqtrade/rpc/api_server/api_schemas.py

@@ -151,6 +151,7 @@ class TradeSchema(BaseModel):
     amount_requested: float
     stake_amount: float
     strategy: str
+    buy_tag: Optional[str]
     timeframe: int
     fee_open: Optional[float]
     fee_open_cost: Optional[float]

freqtrade/rpc/api_server/api_v1.py

@@ -199,8 +199,8 @@ def pair_history(pair: str, timeframe: str, timerange: str, strategy: str,
                  config=Depends(get_config)):
     config = deepcopy(config)
     config.update({
-            'strategy': strategy,
+        'strategy': strategy,
-        })
+    })
     return RPC._rpc_analysed_history_full(config, pair, timeframe, timerange)
 
 
@@ -223,11 +223,11 @@ def list_strategies(config=Depends(get_config)):
 @router.get('/strategy/{strategy}', response_model=StrategyResponse, tags=['strategy'])
 def get_strategy(strategy: str, config=Depends(get_config)):
 
-    config = deepcopy(config)
+    config_ = deepcopy(config)
     from freqtrade.resolvers.strategy_resolver import StrategyResolver
     try:
-        strategy_obj = StrategyResolver._load_strategy(strategy, config,
+        strategy_obj = StrategyResolver._load_strategy(strategy, config_,
-                                                       extra_dir=config.get('strategy_path'))
+                                                       extra_dir=config_.get('strategy_path'))
     except OperationalException:
         raise HTTPException(status_code=404, detail='Strategy not found')
 

freqtrade/rpc/api_server/uvicorn_threaded.py

@@ -32,8 +32,11 @@ class UvicornServer(uvicorn.Server):
             asyncio_setup()
         else:
             asyncio.set_event_loop(uvloop.new_event_loop())
-
+        try:
-        loop = asyncio.get_event_loop()
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            # When running in a thread, we'll not have an eventloop yet.
+            loop = asyncio.new_event_loop()
         loop.run_until_complete(self.serve(sockets=sockets))
 
     @contextlib.contextmanager

freqtrade/rpc/api_server/web_ui.py

@@ -29,6 +29,16 @@ async def ui_version():
     }
 
 
+def is_relative_to(path, base) -> bool:
+    # Helper function simulating behaviour of is_relative_to, which was only added in python 3.9
+    try:
+        path.relative_to(base)
+        return True
+    except ValueError:
+        pass
+    return False
+
+
 @router_ui.get('/{rest_of_path:path}', include_in_schema=False)
 async def index_html(rest_of_path: str):
     """
@@ -37,8 +47,11 @@ async def index_html(rest_of_path: str):
     if rest_of_path.startswith('api') or rest_of_path.startswith('.'):
         raise HTTPException(status_code=404, detail="Not Found")
     uibase = Path(__file__).parent / 'ui/installed/'
-    if (uibase / rest_of_path).is_file():
+    filename = uibase / rest_of_path
-        return FileResponse(str(uibase / rest_of_path))
+    # It's security relevant to check "relative_to".
+    # Without this, Directory-traversal is possible.
+    if filename.is_file() and is_relative_to(filename, uibase):
+        return FileResponse(str(filename))
 
     index_file = uibase / 'index.html'
     if not index_file.is_file():

freqtrade/rpc/fiat_convert.py

@@ -5,7 +5,7 @@ e.g BTC to USD
 
 import datetime
 import logging
-from typing import Dict
+from typing import Dict, List
 
 from cachetools.ttl import TTLCache
 from pycoingecko import CoinGeckoAPI
@@ -25,8 +25,7 @@ class CryptoToFiatConverter:
     """
     __instance = None
     _coingekko: CoinGeckoAPI = None
-
+    _coinlistings: List[Dict] = []
-    _cryptomap: Dict = {}
     _backoff: float = 0.0
 
     def __new__(cls):
@@ -49,9 +48,8 @@ class CryptoToFiatConverter:
 
     def _load_cryptomap(self) -> None:
         try:
-            coinlistings = self._coingekko.get_coins_list()
+            # Use list-comprehension to ensure we get a list.
-            # Create mapping table from symbol to coingekko_id
+            self._coinlistings = [x for x in self._coingekko.get_coins_list()]
-            self._cryptomap = {x['symbol']: x['id'] for x in coinlistings}
         except RequestException as request_exception:
             if "429" in str(request_exception):
                 logger.warning(
@@ -62,13 +60,31 @@ class CryptoToFiatConverter:
             # If the request is not a 429 error we want to raise the normal error
             logger.error(
                 "Could not load FIAT Cryptocurrency map for the following problem: {}".format(
-                  request_exception
+                    request_exception
                 )
             )
         except (Exception) as exception:
             logger.error(
                 f"Could not load FIAT Cryptocurrency map for the following problem: {exception}")
 
+    def _get_gekko_id(self, crypto_symbol):
+        if not self._coinlistings:
+            if self._backoff <= datetime.datetime.now().timestamp():
+                self._load_cryptomap()
+                # Still not loaded.
+                if not self._coinlistings:
+                    return None
+            else:
+                return None
+        found = [x for x in self._coinlistings if x['symbol'] == crypto_symbol]
+        if len(found) == 1:
+            return found[0]['id']
+
+        if len(found) > 0:
+            # Wrong!
+            logger.warning(f"Found multiple mappings in goingekko for {crypto_symbol}.")
+            return None
+
     def convert_amount(self, crypto_amount: float, crypto_symbol: str, fiat_symbol: str) -> float:
         """
         Convert an amount of crypto-currency to fiat
@@ -143,22 +159,14 @@ class CryptoToFiatConverter:
         if crypto_symbol == fiat_symbol:
             return 1.0
 
-        if self._cryptomap == {}:
+        _gekko_id = self._get_gekko_id(crypto_symbol)
-            if self._backoff <= datetime.datetime.now().timestamp():
-                self._load_cryptomap()
-                # return 0.0 if we still don't have data to check, no reason to proceed
-                if self._cryptomap == {}:
-                    return 0.0
-            else:
-                return 0.0
 
-        if crypto_symbol not in self._cryptomap:
+        if not _gekko_id:
             # return 0 for unsupported stake currencies (fiat-convert should not break the bot)
             logger.warning("unsupported crypto-symbol %s - returning 0.0", crypto_symbol)
             return 0.0
 
         try:
-            _gekko_id = self._cryptomap[crypto_symbol]
             return float(
                 self._coingekko.get_price(
                     ids=_gekko_id,

freqtrade/rpc/rpc.py

@@ -557,7 +557,7 @@ class RPC:
                 current_rate = self._freqtrade.exchange.get_rate(
                     trade.pair, refresh=False, side="sell")
                 sell_reason = SellCheckTuple(sell_type=SellType.FORCE_SELL)
-                self._freqtrade.execute_sell(trade, current_rate, sell_reason)
+                self._freqtrade.execute_trade_exit(trade, current_rate, sell_reason)
         # ---- EOF def _exec_forcesell ----
 
         if self._freqtrade.state != State.RUNNING:
@@ -613,7 +613,7 @@ class RPC:
         stakeamount = self._freqtrade.wallets.get_trade_stake_amount(pair)
 
         # execute buy
-        if self._freqtrade.execute_buy(pair, stakeamount, price, forcebuy=True):
+        if self._freqtrade.execute_entry(pair, stakeamount, price, forcebuy=True):
             Trade.commit()
             trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()
             return trade
@@ -776,7 +776,7 @@ class RPC:
         if has_content:
 
             dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].view(int64) // 1000 // 1000
-            # Move open to seperate column when signal for easy plotting
+            # Move open to separate column when signal for easy plotting
             if 'buy' in dataframe.columns:
                 buy_mask = (dataframe['buy'] == 1)
                 buy_signals = int(buy_mask.sum())

freqtrade/rpc/rpc_manager.py

@@ -15,6 +15,7 @@ class RPCManager:
     """
     Class to manage RPC objects (Telegram, API, ...)
     """
+
     def __init__(self, freqtrade) -> None:
         """ Initializes all enabled rpc modules """
         self.registered_modules: List[RPCHandler] = []

freqtrade/rpc/telegram.py

@@ -77,7 +77,6 @@ class Telegram(RPCHandler):
     """  This class handles all telegram communication """
 
     def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None:
-
         """
         Init the Telegram call, and init the super class RPCHandler
         :param rpc: instance of RPC Helper class
@@ -208,15 +207,25 @@ class Telegram(RPCHandler):
         else:
             msg['stake_amount_fiat'] = 0
 
-        message = (f"\N{LARGE BLUE CIRCLE} *{msg['exchange']}:* Buying {msg['pair']}"
+        content = []
-                   f" (#{msg['trade_id']})\n"
+        content.append(
-                   f"*Amount:* `{msg['amount']:.8f}`\n"
+            f"\N{LARGE BLUE CIRCLE} *{msg['exchange']}:* Buying {msg['pair']}"
-                   f"*Open Rate:* `{msg['limit']:.8f}`\n"
+            f" (#{msg['trade_id']})\n"
-                   f"*Current Rate:* `{msg['current_rate']:.8f}`\n"
+        )
-                   f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}")
+        if msg.get('buy_tag', None):
-
+            content.append(f"*Buy Tag:* `{msg['buy_tag']}`\n")
+        content.append(f"*Amount:* `{msg['amount']:.8f}`\n")
+        content.append(f"*Open Rate:* `{msg['limit']:.8f}`\n")
+        content.append(f"*Current Rate:* `{msg['current_rate']:.8f}`\n")
+        content.append(
+            f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}"
+        )
         if msg.get('fiat_currency', None):
-            message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}"
+            content.append(
+                f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}"
+            )
+
+        message = ''.join(content)
         message += ")`"
         return message
 
@@ -260,7 +269,7 @@ class Telegram(RPCHandler):
         noti = ''
         if msg_type == RPCMessageType.SELL:
             sell_noti = self._config['telegram'] \
-              .get('notification_settings', {}).get(str(msg_type), {})
+                .get('notification_settings', {}).get(str(msg_type), {})
             # For backward compatibility sell still can be string
             if isinstance(sell_noti, str):
                 noti = sell_noti
@@ -268,7 +277,7 @@ class Telegram(RPCHandler):
                 noti = sell_noti.get(str(msg['sell_reason']), default_noti)
         else:
             noti = self._config['telegram'] \
-              .get('notification_settings', {}).get(str(msg_type), default_noti)
+                .get('notification_settings', {}).get(str(msg_type), default_noti)
 
         if noti == 'off':
             logger.info(f"Notification '{msg_type}' not sent.")
@@ -354,6 +363,7 @@ class Telegram(RPCHandler):
                     "*Trade ID:* `{trade_id}` `(since {open_date_hum})`",
                     "*Current Pair:* {pair}",
                     "*Amount:* `{amount} ({stake_amount} {base_currency})`",
+                    "*Buy Tag:* `{buy_tag}`" if r['buy_tag'] else "",
                     "*Open Rate:* `{open_rate:.8f}`",
                     "*Close Rate:* `{close_rate}`" if r['close_rate'] else "",
                     "*Current Rate:* `{current_rate:.8f}`",
@@ -530,7 +540,7 @@ class Telegram(RPCHandler):
                 f"`{first_trade_date}`\n"
                 f"*Latest Trade opened:* `{latest_trade_date}\n`"
                 f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`"
-                )
+            )
             if stats['closed_trade_count'] > 0:
                 markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n"
                                  f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`")
@@ -565,13 +575,14 @@ class Telegram(RPCHandler):
         sell_reasons_msg = tabulate(
             sell_reasons_tabulate,
             headers=['Sell Reason', 'Sells', 'Wins', 'Losses']
-            )
+        )
         durations = stats['durations']
-        duration_msg = tabulate([
+        duration_msg = tabulate(
-            ['Wins', str(timedelta(seconds=durations['wins']))
+            [
-             if durations['wins'] != 'N/A' else 'N/A'],
+                ['Wins', str(timedelta(seconds=durations['wins']))
-            ['Losses', str(timedelta(seconds=durations['losses']))
+                 if durations['wins'] != 'N/A' else 'N/A'],
-             if durations['losses'] != 'N/A' else 'N/A']
+                ['Losses', str(timedelta(seconds=durations['losses']))
+                 if durations['losses'] != 'N/A' else 'N/A']
             ],
             headers=['', 'Avg. Duration']
         )
@@ -1089,7 +1100,7 @@ class Telegram(RPCHandler):
         if reload_able:
             reply_markup = InlineKeyboardMarkup([
                 [InlineKeyboardButton("Refresh", callback_data=callback_path)],
-                ])
+            ])
         else:
             reply_markup = InlineKeyboardMarkup([[]])
         msg += "\nUpdated: {}".format(datetime.now().ctime())

freqtrade/strategy/__init__.py

@@ -1,7 +1,7 @@
 # flake8: noqa: F401
 from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date,
                                 timeframe_to_prev_date, timeframe_to_seconds)
-from freqtrade.strategy.hyper import (CategoricalParameter, DecimalParameter, IntParameter,
+from freqtrade.strategy.hyper import (BooleanParameter, CategoricalParameter, DecimalParameter,
-                                      RealParameter)
+                                      IntParameter, RealParameter)
 from freqtrade.strategy.interface import IStrategy
 from freqtrade.strategy.strategy_helper import merge_informative_pair, stoploss_from_open

freqtrade/strategy/hyper.py

@@ -270,6 +270,28 @@ class CategoricalParameter(BaseParameter):
             return [self.value]
 
 
+class BooleanParameter(CategoricalParameter):
+
+    def __init__(self, *, default: Optional[Any] = None,
+                 space: Optional[str] = None, optimize: bool = True, load: bool = True, **kwargs):
+        """
+        Initialize hyperopt-optimizable Boolean Parameter.
+        It's a shortcut to `CategoricalParameter([True, False])`.
+        :param default: A default value. If not specified, first item from specified space will be
+         used.
+        :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if
+         parameter field
+         name is prefixed with 'buy_' or 'sell_'.
+        :param optimize: Include parameter in hyperopt optimizations.
+        :param load: Load parameter value from {space}_params.
+        :param kwargs: Extra parameters to skopt.space.Categorical.
+        """
+
+        categories = [True, False]
+        super().__init__(categories=categories, default=default, space=space, optimize=optimize,
+                         load=load, **kwargs)
+
+
 class HyperStrategyMixin(object):
     """
     A helper base class which allows HyperOptAuto class to reuse implementations of buy/sell
@@ -283,6 +305,7 @@ class HyperStrategyMixin(object):
         self.config = config
         self.ft_buy_params: List[BaseParameter] = []
         self.ft_sell_params: List[BaseParameter] = []
+        self.ft_protection_params: List[BaseParameter] = []
 
         self._load_hyper_params(config.get('runmode') == RunMode.HYPEROPT)
 
@@ -292,11 +315,12 @@ class HyperStrategyMixin(object):
         :param category:
         :return:
         """
-        if category not in ('buy', 'sell', None):
+        if category not in ('buy', 'sell', 'protection', None):
-            raise OperationalException('Category must be one of: "buy", "sell", None.')
+            raise OperationalException(
+                'Category must be one of: "buy", "sell", "protection", None.')
 
         if category is None:
-            params = self.ft_buy_params + self.ft_sell_params
+            params = self.ft_buy_params + self.ft_sell_params + self.ft_protection_params
         else:
             params = getattr(self, f"ft_{category}_params")
 
@@ -324,9 +348,10 @@ class HyperStrategyMixin(object):
         params: Dict = {
             'buy': list(cls.detect_parameters('buy')),
             'sell': list(cls.detect_parameters('sell')),
+            'protection': list(cls.detect_parameters('protection')),
         }
         params.update({
-            'count': len(params['buy'] + params['sell'])
+            'count': len(params['buy'] + params['sell'] + params['protection'])
         })
 
         return params
@@ -340,9 +365,12 @@ class HyperStrategyMixin(object):
         self._ft_params_from_file = params
         buy_params = deep_merge_dicts(params.get('buy', {}), getattr(self, 'buy_params', {}))
         sell_params = deep_merge_dicts(params.get('sell', {}), getattr(self, 'sell_params', {}))
+        protection_params = deep_merge_dicts(params.get('protection', {}),
+                                             getattr(self, 'protection_params', {}))
 
         self._load_params(buy_params, 'buy', hyperopt)
         self._load_params(sell_params, 'sell', hyperopt)
+        self._load_params(protection_params, 'protection', hyperopt)
 
     def load_params_from_file(self) -> Dict:
         filename_str = getattr(self, '__file__', '')
@@ -397,7 +425,8 @@ class HyperStrategyMixin(object):
         """
         params = {
             'buy': {},
-            'sell': {}
+            'sell': {},
+            'protection': {},
         }
         for name, p in self.enumerate_parameters():
             if not p.optimize or not p.in_space:

freqtrade/strategy/interface.py

@@ -13,7 +13,7 @@ from pandas import DataFrame
 
 from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.data.dataprovider import DataProvider
-from freqtrade.enums import SellType, SignalType
+from freqtrade.enums import SellType, SignalTagType, SignalType
 from freqtrade.exceptions import OperationalException, StrategyError
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
 from freqtrade.exchange.exchange import timeframe_to_next_date
@@ -280,6 +280,43 @@ class IStrategy(ABC, HyperStrategyMixin):
         """
         return self.stoploss
 
+    def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float,
+                           **kwargs) -> float:
+        """
+        Custom entry price logic, returning the new entry price.
+
+        For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
+
+        When not implemented by a strategy, returns None, orderbook is used to set entry price
+
+        :param pair: Pair that's currently analyzed
+        :param current_time: datetime object, containing the current datetime
+        :param proposed_rate: Rate, calculated based on pricing settings in ask_strategy.
+        :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+        :return float: New entry price value if provided
+        """
+        return proposed_rate
+
+    def custom_exit_price(self, pair: str, trade: Trade,
+                          current_time: datetime, proposed_rate: float,
+                          current_profit: float, **kwargs) -> float:
+        """
+        Custom exit price logic, returning the new exit price.
+
+        For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
+
+        When not implemented by a strategy, returns None, orderbook is used to set exit price
+
+        :param pair: Pair that's currently analyzed
+        :param trade: trade object.
+        :param current_time: datetime object, containing the current datetime
+        :param proposed_rate: Rate, calculated based on pricing settings in ask_strategy.
+        :param current_profit: Current profit (as ratio), calculated based on current_rate.
+        :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+        :return float: New exit price value if provided
+        """
+        return proposed_rate
+
     def custom_sell(self, pair: str, trade: Trade, current_time: datetime, current_rate: float,
                     current_profit: float, **kwargs) -> Optional[Union[str, bool]]:
         """
@@ -288,10 +325,10 @@ class IStrategy(ABC, HyperStrategyMixin):
         time. This method is not called when sell signal is set.
 
         This method should be overridden to create sell signals that depend on trade parameters. For
-        example you could implement a stoploss relative to candle when trade was opened, or a custom
+        example you could implement a sell relative to the candle when the trade was opened,
-        1:2 risk-reward ROI.
+        or a custom 1:2 risk-reward ROI.
 
-        Custom sell reason max length is 64. Exceeding this limit will raise OperationalException.
+        Custom sell reason max length is 64. Exceeding characters will be removed.
 
         :param pair: Pair that's currently analyzed
         :param trade: trade object.
@@ -422,6 +459,7 @@ class IStrategy(ABC, HyperStrategyMixin):
             logger.debug("Skipping TA Analysis for already analyzed candle")
             dataframe['buy'] = 0
             dataframe['sell'] = 0
+            dataframe['buy_tag'] = None
 
         # Other Defs in strategy that want to be called every loop here
         # twitter_sell = self.watch_twitter_feed(dataframe, metadata)
@@ -482,8 +520,6 @@ class IStrategy(ABC, HyperStrategyMixin):
             message = "No dataframe returned (return statement missing?)."
         elif 'buy' not in dataframe:
             message = "Buy column not set."
-        elif 'sell' not in dataframe:
-            message = "Sell column not set."
         elif df_len != len(dataframe):
             message = message_template.format("length")
         elif df_close != dataframe["close"].iloc[-1]:
@@ -496,7 +532,12 @@ class IStrategy(ABC, HyperStrategyMixin):
             else:
                 raise StrategyError(message)
 
-    def get_signal(self, pair: str, timeframe: str, dataframe: DataFrame) -> Tuple[bool, bool]:
+    def get_signal(
+        self,
+        pair: str,
+        timeframe: str,
+        dataframe: DataFrame
+    ) -> Tuple[bool, bool, Optional[str]]:
         """
         Calculates current signal based based on the buy / sell columns of the dataframe.
         Used by Bot to get the signal to buy or sell
@@ -507,7 +548,7 @@ class IStrategy(ABC, HyperStrategyMixin):
         """
         if not isinstance(dataframe, DataFrame) or dataframe.empty:
             logger.warning(f'Empty candle (OHLCV) data for pair {pair}')
-            return False, False
+            return False, False, None
 
         latest_date = dataframe['date'].max()
         latest = dataframe.loc[dataframe['date'] == latest_date].iloc[-1]
@@ -522,9 +563,16 @@ class IStrategy(ABC, HyperStrategyMixin):
                 'Outdated history for pair %s. Last tick is %s minutes old',
                 pair, int((arrow.utcnow() - latest_date).total_seconds() // 60)
             )
-            return False, False
+            return False, False, None
+
+        buy = latest[SignalType.BUY.value] == 1
+
+        sell = False
+        if SignalType.SELL.value in latest:
+            sell = latest[SignalType.SELL.value] == 1
+
+        buy_tag = latest.get(SignalTagType.BUY_TAG.value, None)
 
-        (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1
         logger.debug('trigger: %s (pair=%s) buy=%s sell=%s',
                      latest['date'], pair, str(buy), str(sell))
         timeframe_seconds = timeframe_to_seconds(timeframe)
@@ -532,8 +580,8 @@ class IStrategy(ABC, HyperStrategyMixin):
                                       current_time=datetime.now(timezone.utc),
                                       timeframe_seconds=timeframe_seconds,
                                       buy=buy):
-            return False, sell
+            return False, sell, buy_tag
-        return buy, sell
+        return buy, sell, buy_tag
 
     def ignore_expired_candle(self, latest_date: datetime, current_time: datetime,
                               timeframe_seconds: int, buy: bool):
@@ -557,7 +605,7 @@ class IStrategy(ABC, HyperStrategyMixin):
         current_rate = rate
         current_profit = trade.calc_profit_ratio(current_rate)
 
-        trade.adjust_min_max_rates(high or current_rate)
+        trade.adjust_min_max_rates(high or current_rate, low or current_rate)
 
         stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade,
                                               current_time=date, current_profit=current_profit,
@@ -721,7 +769,7 @@ class IStrategy(ABC, HyperStrategyMixin):
         else:
             return current_profit > roi
 
-    def ohlcvdata_to_dataframe(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]:
+    def advise_all_indicators(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]:
         """
         Populates indicators for given candle (OHLCV) data (for multiple pairs)
         Does not run advise_buy or advise_sell!

freqtrade/strategy/strategy_helper.py

@@ -38,7 +38,7 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame,
         # Detailed explanation in https://github.com/freqtrade/freqtrade/issues/4073
         informative['date_merge'] = (
             informative["date"] + pd.to_timedelta(minutes_inf, 'm') - pd.to_timedelta(minutes, 'm')
-            )
+        )
     else:
         raise ValueError("Tried to merge a faster timeframe to a slower timeframe."
                          "This would create new rows, and can throw off your regular indicators.")

freqtrade/templates/base_config.json.j2

@@ -25,7 +25,7 @@
     "ask_strategy": {
         "price_side": "ask",
         "use_order_book": true,
-        "order_book_top": 1,
+        "order_book_top": 1
     },
     {{ exchange | indent(4) }},
     "pairlists": [

freqtrade/templates/base_strategy.py.j2

@@ -6,8 +6,8 @@ import numpy as np  # noqa
 import pandas as pd  # noqa
 from pandas import DataFrame
 
-from freqtrade.strategy import IStrategy
+from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter,
-from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter
+                                IStrategy, IntParameter)
 
 # --------------------------------
 # Add your lib to import here

freqtrade/templates/sample_strategy.py

@@ -6,8 +6,8 @@ import numpy as np  # noqa
 import pandas as pd  # noqa
 from pandas import DataFrame
 
-from freqtrade.strategy import IStrategy
+from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter,
-from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter
+                                IStrategy, IntParameter)
 
 # --------------------------------
 # Add your lib to import here

freqtrade/templates/subtemplates/strategy_methods_advanced.j2

@@ -12,6 +12,23 @@ def bot_loop_start(self, **kwargs) -> None:
     """
     pass
 
+def custom_stake_amount(self, pair: str, current_time: 'datetime', current_rate: float,
+                        proposed_stake: float, min_stake: float, max_stake: float,
+                        **kwargs) -> float:
+    """
+    Customize stake size for each new trade. This method is not called when edge module is
+    enabled.
+
+    :param pair: Pair that's currently analyzed
+    :param current_time: datetime object, containing the current datetime
+    :param current_rate: Rate, calculated based on pricing settings in ask_strategy.
+    :param proposed_stake: A stake amount proposed by the bot.
+    :param min_stake: Minimal stake size allowed by exchange.
+    :param max_stake: Balance available for trading.
+    :return: A stake size, which is between min_stake and max_stake.
+    """
+    return proposed_stake
+
 use_custom_stoploss = True
 
 def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime',
@@ -38,6 +55,30 @@ def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime',
     """
     return self.stoploss
 
+def custom_sell(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float,
+                current_profit: float, **kwargs) -> 'Optional[Union[str, bool]]':
+    """
+    Custom sell signal logic indicating that specified position should be sold. Returning a
+    string or True from this method is equal to setting sell signal on a candle at specified
+    time. This method is not called when sell signal is set.
+
+    This method should be overridden to create sell signals that depend on trade parameters. For
+    example you could implement a sell relative to the candle when the trade was opened,
+    or a custom 1:2 risk-reward ROI.
+
+    Custom sell reason max length is 64. Exceeding characters will be removed.
+
+    :param pair: Pair that's currently analyzed
+    :param trade: trade object.
+    :param current_time: datetime object, containing the current datetime
+    :param current_rate: Rate, calculated based on pricing settings in ask_strategy.
+    :param current_profit: Current profit (as ratio), calculated based on current_rate.
+    :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+    :return: To execute sell, return a string with custom sell reason or True. Otherwise return
+    None or False.
+    """
+    return None
+
 def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float,
                         time_in_force: str, current_time: 'datetime', **kwargs) -> bool:
     """

requirements-dev.txt

@@ -6,20 +6,20 @@
 coveralls==3.2.0
 flake8==3.9.2
 flake8-type-annotations==0.1.0
-flake8-tidy-imports==4.3.0
+flake8-tidy-imports==4.4.1
 mypy==0.910
 pytest==6.2.4
 pytest-asyncio==0.15.1
 pytest-cov==2.12.1
 pytest-mock==3.6.1
 pytest-random-order==1.0.4
-isort==5.9.2
+isort==5.9.3
 
 # Convert jupyter notebooks to markdown documents
 nbconvert==6.1.0
 
 # mypy types
-types-cachetools==0.1.9
+types-cachetools==4.2.0
-types-filelock==0.1.4
+types-filelock==0.1.5
-types-requests==2.25.0
+types-requests==2.25.6
-types-tabulate==0.1.1
+types-tabulate==0.8.2

requirements-hyperopt.txt

@@ -2,7 +2,7 @@
 -r requirements.txt
 
 # Required for hyperopt
-scipy==1.7.0
+scipy==1.7.1
 scikit-learn==0.24.2
 scikit-optimize==0.8.1
 filelock==3.0.12

requirements-plot.txt

@@ -1,5 +1,5 @@
 # Include all requirements to run the bot.
 -r requirements.txt
 
-plotly==5.1.0
+plotly==5.2.1
 

requirements.txt

@@ -1,11 +1,11 @@
-numpy==1.21.1
+numpy==1.21.2
-pandas==1.3.1
+pandas==1.3.2
 
-ccxt==1.53.72
+ccxt==1.55.28
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==3.4.7
 aiohttp==3.7.4.post0
-SQLAlchemy==1.4.22
+SQLAlchemy==1.4.23
 python-telegram-bot==13.7
 arrow==1.1.1
 cachetools==4.2.2
@@ -31,8 +31,8 @@ python-rapidjson==1.4
 sdnotify==0.3.2
 
 # API Server
-fastapi==0.67.0
+fastapi==0.68.0
-uvicorn==0.14.0
+uvicorn==0.15.0
 pyjwt==2.1.0
 aiofiles==0.7.0
 
@@ -40,4 +40,4 @@ aiofiles==0.7.0
 colorama==0.4.4
 # Building config files interactively
 questionary==1.10.0
-prompt-toolkit==3.0.19
+prompt-toolkit==3.0.20

setup.sh

@@ -163,7 +163,7 @@ function update() {
 # Reset Develop or Stable branch
 function reset() {
     echo "----------------------------"
-    echo "Reseting branch and virtual env"
+    echo "Resetting branch and virtual env"
     echo "----------------------------"
 
     if [ "1" == $(git branch -vv |grep -cE "\* develop|\* stable") ]

tests/commands/test_commands.py

@@ -510,17 +510,6 @@ def test_start_new_strategy(mocker, caplog):
         start_new_strategy(get_args(args))
 
 
-def test_start_new_strategy_DefaultStrat(mocker, caplog):
-    args = [
-        "new-strategy",
-        "--strategy",
-        "DefaultStrategy"
-    ]
-    with pytest.raises(OperationalException,
-                       match=r"DefaultStrategy is not allowed as name\."):
-        start_new_strategy(get_args(args))
-
-
 def test_start_new_strategy_no_arg(mocker, caplog):
     args = [
         "new-strategy",
@@ -552,17 +541,6 @@ def test_start_new_hyperopt(mocker, caplog):
         start_new_hyperopt(get_args(args))
 
 
-def test_start_new_hyperopt_DefaultHyperopt(mocker, caplog):
-    args = [
-        "new-hyperopt",
-        "--hyperopt",
-        "DefaultHyperopt"
-    ]
-    with pytest.raises(OperationalException,
-                       match=r"DefaultHyperopt is not allowed as name\."):
-        start_new_hyperopt(get_args(args))
-
-
 def test_start_new_hyperopt_no_arg(mocker):
     args = [
         "new-hyperopt",
@@ -827,9 +805,9 @@ def test_start_list_strategies(mocker, caplog, capsys):
     # pargs['config'] = None
     start_list_strategies(pargs)
     captured = capsys.readouterr()
-    assert "TestStrategyLegacy" in captured.out
+    assert "TestStrategyLegacyV1" in captured.out
-    assert "legacy_strategy.py" not in captured.out
+    assert "legacy_strategy_v1.py" not in captured.out
-    assert "DefaultStrategy" in captured.out
+    assert "StrategyTestV2" in captured.out
 
     # Test regular output
     args = [
@@ -842,9 +820,9 @@ def test_start_list_strategies(mocker, caplog, capsys):
     # pargs['config'] = None
     start_list_strategies(pargs)
     captured = capsys.readouterr()
-    assert "TestStrategyLegacy" in captured.out
+    assert "TestStrategyLegacyV1" in captured.out
-    assert "legacy_strategy.py" in captured.out
+    assert "legacy_strategy_v1.py" in captured.out
-    assert "DefaultStrategy" in captured.out
+    assert "StrategyTestV2" in captured.out
 
 
 def test_start_list_hyperopts(mocker, caplog, capsys):
@@ -861,7 +839,7 @@ def test_start_list_hyperopts(mocker, caplog, capsys):
     captured = capsys.readouterr()
     assert "TestHyperoptLegacy" not in captured.out
     assert "legacy_hyperopt.py" not in captured.out
-    assert "DefaultHyperOpt" in captured.out
+    assert "HyperoptTestSepFile" in captured.out
     assert "test_hyperopt.py" not in captured.out
 
     # Test regular output
@@ -876,7 +854,7 @@ def test_start_list_hyperopts(mocker, caplog, capsys):
     captured = capsys.readouterr()
     assert "TestHyperoptLegacy" not in captured.out
     assert "legacy_hyperopt.py" not in captured.out
-    assert "DefaultHyperOpt" in captured.out
+    assert "HyperoptTestSepFile" in captured.out
 
 
 def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys):
@@ -938,247 +916,261 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys):
         pytest.fail(f'Expected well formed JSON, but failed to parse: {captured.out}')
 
 
-def test_hyperopt_list(mocker, capsys, caplog, saved_hyperopt_results,
+def test_hyperopt_list(mocker, capsys, caplog, saved_hyperopt_results, tmpdir):
-                       saved_hyperopt_results_legacy, tmpdir):
     csv_file = Path(tmpdir) / "test.csv"
-    for res in (saved_hyperopt_results, saved_hyperopt_results_legacy):
+    mocker.patch(
-        mocker.patch(
+        'freqtrade.optimize.hyperopt_tools.HyperoptTools._test_hyperopt_results_exist',
-            'freqtrade.optimize.hyperopt_tools.HyperoptTools.load_previous_results',
+        return_value=True
-            MagicMock(return_value=res)
         )
 
-        args = [
+    def fake_iterator(*args, **kwargs):
-            "hyperopt-list",
+        yield from [saved_hyperopt_results]
-            "--no-details",
+
-            "--no-color",
+    mocker.patch(
-        ]
+        'freqtrade.optimize.hyperopt_tools.HyperoptTools._read_results',
-        pargs = get_args(args)
+        side_effect=fake_iterator
-        pargs['config'] = None
+    )
-        start_hyperopt_list(pargs)
+
-        captured = capsys.readouterr()
+    args = [
-        assert all(x in captured.out
+        "hyperopt-list",
-                   for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12",
+        "--no-details",
-                             " 6/12", " 7/12", " 8/12", " 9/12", " 10/12",
+        "--no-color",
-                             " 11/12", " 12/12"])
+    ]
-        args = [
+    pargs = get_args(args)
-            "hyperopt-list",
+    pargs['config'] = None
-            "--best",
+    start_hyperopt_list(pargs)
-            "--no-details",
+    captured = capsys.readouterr()
-            "--no-color",
+    assert all(x in captured.out
-        ]
+               for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12",
-        pargs = get_args(args)
+                         " 6/12", " 7/12", " 8/12", " 9/12", " 10/12",
-        pargs['config'] = None
+                         " 11/12", " 12/12"])
-        start_hyperopt_list(pargs)
+    args = [
-        captured = capsys.readouterr()
+        "hyperopt-list",
-        assert all(x in captured.out
+        "--best",
-                   for x in [" 1/12", " 5/12", " 10/12"])
+        "--no-details",
-        assert all(x not in captured.out
+        "--no-color",
-                   for x in [" 2/12", " 3/12", " 4/12", " 6/12", " 7/12", " 8/12", " 9/12",
+    ]
-                             " 11/12", " 12/12"])
+    pargs = get_args(args)
-        args = [
+    pargs['config'] = None
-            "hyperopt-list",
+    start_hyperopt_list(pargs)
-            "--profitable",
+    captured = capsys.readouterr()
-            "--no-details",
+    assert all(x in captured.out
-            "--no-color",
+               for x in [" 1/12", " 5/12", " 10/12"])
-        ]
+    assert all(x not in captured.out
-        pargs = get_args(args)
+               for x in [" 2/12", " 3/12", " 4/12", " 6/12", " 7/12", " 8/12", " 9/12",
-        pargs['config'] = None
+                         " 11/12", " 12/12"])
-        start_hyperopt_list(pargs)
+    args = [
-        captured = capsys.readouterr()
+        "hyperopt-list",
-        assert all(x in captured.out
+        "--profitable",
-                   for x in [" 2/12", " 10/12"])
+        "--no-details",
-        assert all(x not in captured.out
+        "--no-color",
-                   for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12",
+    ]
-                             " 11/12", " 12/12"])
+    pargs = get_args(args)
-        args = [
+    pargs['config'] = None
-            "hyperopt-list",
+    start_hyperopt_list(pargs)
-            "--profitable",
+    captured = capsys.readouterr()
-            "--no-color",
+    assert all(x in captured.out
-        ]
+               for x in [" 2/12", " 10/12"])
-        pargs = get_args(args)
+    assert all(x not in captured.out
-        pargs['config'] = None
+               for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12",
-        start_hyperopt_list(pargs)
+                         " 11/12", " 12/12"])
-        captured = capsys.readouterr()
+    args = [
-        assert all(x in captured.out
+        "hyperopt-list",
-                   for x in [" 2/12", " 10/12", "Best result:", "Buy hyperspace params",
+        "--profitable",
-                             "Sell hyperspace params", "ROI table", "Stoploss"])
+        "--no-color",
-        assert all(x not in captured.out
+    ]
-                   for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12",
+    pargs = get_args(args)
-                             " 11/12", " 12/12"])
+    pargs['config'] = None
-        args = [
+    start_hyperopt_list(pargs)
-            "hyperopt-list",
+    captured = capsys.readouterr()
-            "--no-details",
+    assert all(x in captured.out
-            "--no-color",
+               for x in [" 2/12", " 10/12", "Best result:", "Buy hyperspace params",
-            "--min-trades", "20",
+                         "Sell hyperspace params", "ROI table", "Stoploss"])
-        ]
+    assert all(x not in captured.out
-        pargs = get_args(args)
+               for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12",
-        pargs['config'] = None
+                         " 11/12", " 12/12"])
-        start_hyperopt_list(pargs)
+    args = [
-        captured = capsys.readouterr()
+        "hyperopt-list",
-        assert all(x in captured.out
+        "--no-details",
-                   for x in [" 3/12", " 6/12", " 7/12", " 9/12", " 11/12"])
+        "--no-color",
-        assert all(x not in captured.out
+        "--min-trades", "20",
-                   for x in [" 1/12", " 2/12", " 4/12", " 5/12", " 8/12", " 10/12", " 12/12"])
+    ]
-        args = [
+    pargs = get_args(args)
-            "hyperopt-list",
+    pargs['config'] = None
-            "--profitable",
+    start_hyperopt_list(pargs)
-            "--no-details",
+    captured = capsys.readouterr()
-            "--no-color",
+    assert all(x in captured.out
-            "--max-trades", "20",
+               for x in [" 3/12", " 6/12", " 7/12", " 9/12", " 11/12"])
-        ]
+    assert all(x not in captured.out
-        pargs = get_args(args)
+               for x in [" 1/12", " 2/12", " 4/12", " 5/12", " 8/12", " 10/12", " 12/12"])
-        pargs['config'] = None
+    args = [
-        start_hyperopt_list(pargs)
+        "hyperopt-list",
-        captured = capsys.readouterr()
+        "--profitable",
-        assert all(x in captured.out
+        "--no-details",
-                   for x in [" 2/12", " 10/12"])
+        "--no-color",
-        assert all(x not in captured.out
+        "--max-trades", "20",
-                   for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12",
+    ]
-                             " 11/12", " 12/12"])
+    pargs = get_args(args)
-        args = [
+    pargs['config'] = None
-            "hyperopt-list",
+    start_hyperopt_list(pargs)
-            "--profitable",
+    captured = capsys.readouterr()
-            "--no-details",
+    assert all(x in captured.out
-            "--no-color",
+               for x in [" 2/12", " 10/12"])
-            "--min-avg-profit", "0.11",
+    assert all(x not in captured.out
-        ]
+               for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12",
-        pargs = get_args(args)
+                         " 11/12", " 12/12"])
-        pargs['config'] = None
+    args = [
-        start_hyperopt_list(pargs)
+        "hyperopt-list",
-        captured = capsys.readouterr()
+        "--profitable",
-        assert all(x in captured.out
+        "--no-details",
-                   for x in [" 2/12"])
+        "--no-color",
-        assert all(x not in captured.out
+        "--min-avg-profit", "0.11",
-                   for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12",
+    ]
-                             " 10/12", " 11/12", " 12/12"])
+    pargs = get_args(args)
-        args = [
+    pargs['config'] = None
-            "hyperopt-list",
+    start_hyperopt_list(pargs)
-            "--no-details",
+    captured = capsys.readouterr()
-            "--no-color",
+    assert all(x in captured.out
-            "--max-avg-profit", "0.10",
+               for x in [" 2/12"])
-        ]
+    assert all(x not in captured.out
-        pargs = get_args(args)
+               for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12",
-        pargs['config'] = None
+                         " 10/12", " 11/12", " 12/12"])
-        start_hyperopt_list(pargs)
+    args = [
-        captured = capsys.readouterr()
+        "hyperopt-list",
-        assert all(x in captured.out
+        "--no-details",
-                   for x in [" 1/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12",
+        "--no-color",
-                             " 11/12"])
+        "--max-avg-profit", "0.10",
-        assert all(x not in captured.out
+    ]
-                   for x in [" 2/12", " 4/12", " 10/12", " 12/12"])
+    pargs = get_args(args)
-        args = [
+    pargs['config'] = None
-            "hyperopt-list",
+    start_hyperopt_list(pargs)
-            "--no-details",
+    captured = capsys.readouterr()
-            "--no-color",
+    assert all(x in captured.out
-            "--min-total-profit", "0.4",
+               for x in [" 1/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12",
-        ]
+                         " 11/12"])
-        pargs = get_args(args)
+    assert all(x not in captured.out
-        pargs['config'] = None
+               for x in [" 2/12", " 4/12", " 10/12", " 12/12"])
-        start_hyperopt_list(pargs)
+    args = [
-        captured = capsys.readouterr()
+        "hyperopt-list",
-        assert all(x in captured.out
+        "--no-details",
-                   for x in [" 10/12"])
+        "--no-color",
-        assert all(x not in captured.out
+        "--min-total-profit", "0.4",
-                   for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12",
+    ]
-                             " 9/12", " 11/12", " 12/12"])
+    pargs = get_args(args)
-        args = [
+    pargs['config'] = None
-            "hyperopt-list",
+    start_hyperopt_list(pargs)
-            "--no-details",
+    captured = capsys.readouterr()
-            "--no-color",
+    assert all(x in captured.out
-            "--max-total-profit", "0.4",
+               for x in [" 10/12"])
-        ]
+    assert all(x not in captured.out
-        pargs = get_args(args)
+               for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12",
-        pargs['config'] = None
+                         " 9/12", " 11/12", " 12/12"])
-        start_hyperopt_list(pargs)
+    args = [
-        captured = capsys.readouterr()
+        "hyperopt-list",
-        assert all(x in captured.out
+        "--no-details",
-                   for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12",
+        "--no-color",
-                             " 9/12", " 11/12"])
+        "--max-total-profit", "0.4",
-        assert all(x not in captured.out
+    ]
-                   for x in [" 4/12", " 10/12", " 12/12"])
+    pargs = get_args(args)
-        args = [
+    pargs['config'] = None
-            "hyperopt-list",
+    start_hyperopt_list(pargs)
-            "--no-details",
+    captured = capsys.readouterr()
-            "--no-color",
+    assert all(x in captured.out
-            "--min-objective", "0.1",
+               for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12",
-        ]
+                         " 9/12", " 11/12"])
-        pargs = get_args(args)
+    assert all(x not in captured.out
-        pargs['config'] = None
+               for x in [" 4/12", " 10/12", " 12/12"])
-        start_hyperopt_list(pargs)
+    args = [
-        captured = capsys.readouterr()
+        "hyperopt-list",
-        assert all(x in captured.out
+        "--no-details",
-                   for x in [" 10/12"])
+        "--no-color",
-        assert all(x not in captured.out
+        "--min-objective", "0.1",
-                   for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12",
+    ]
-                             " 9/12", " 11/12", " 12/12"])
+    pargs = get_args(args)
-        args = [
+    pargs['config'] = None
-            "hyperopt-list",
+    start_hyperopt_list(pargs)
-            "--no-details",
+    captured = capsys.readouterr()
-            "--max-objective", "0.1",
+    assert all(x in captured.out
-        ]
+               for x in [" 10/12"])
-        pargs = get_args(args)
+    assert all(x not in captured.out
-        pargs['config'] = None
+               for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12",
-        start_hyperopt_list(pargs)
+                         " 9/12", " 11/12", " 12/12"])
-        captured = capsys.readouterr()
+    args = [
-        assert all(x in captured.out
+        "hyperopt-list",
-                   for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12",
+        "--no-details",
-                             " 9/12", " 11/12"])
+        "--max-objective", "0.1",
-        assert all(x not in captured.out
+    ]
-                   for x in [" 4/12", " 10/12", " 12/12"])
+    pargs = get_args(args)
-        args = [
+    pargs['config'] = None
-            "hyperopt-list",
+    start_hyperopt_list(pargs)
-            "--profitable",
+    captured = capsys.readouterr()
-            "--no-details",
+    assert all(x in captured.out
-            "--no-color",
+               for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12",
-            "--min-avg-time", "2000",
+                         " 9/12", " 11/12"])
-        ]
+    assert all(x not in captured.out
-        pargs = get_args(args)
+               for x in [" 4/12", " 10/12", " 12/12"])
-        pargs['config'] = None
+    args = [
-        start_hyperopt_list(pargs)
+        "hyperopt-list",
-        captured = capsys.readouterr()
+        "--profitable",
-        assert all(x in captured.out
+        "--no-details",
-                   for x in [" 10/12"])
+        "--no-color",
-        assert all(x not in captured.out
+        "--min-avg-time", "2000",
-                   for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12",
+    ]
-                             " 8/12", " 9/12", " 11/12", " 12/12"])
+    pargs = get_args(args)
-        args = [
+    pargs['config'] = None
-            "hyperopt-list",
+    start_hyperopt_list(pargs)
-            "--no-details",
+    captured = capsys.readouterr()
-            "--no-color",
+    assert all(x in captured.out
-            "--max-avg-time", "1500",
+               for x in [" 10/12"])
-        ]
+    assert all(x not in captured.out
-        pargs = get_args(args)
+               for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12",
-        pargs['config'] = None
+                         " 8/12", " 9/12", " 11/12", " 12/12"])
-        start_hyperopt_list(pargs)
+    args = [
-        captured = capsys.readouterr()
+        "hyperopt-list",
-        assert all(x in captured.out
+        "--no-details",
-                   for x in [" 2/12", " 6/12"])
+        "--no-color",
-        assert all(x not in captured.out
+        "--max-avg-time", "1500",
-                   for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 7/12", " 8/12"
+    ]
-                             " 9/12", " 10/12", " 11/12", " 12/12"])
+    pargs = get_args(args)
-        args = [
+    pargs['config'] = None
-            "hyperopt-list",
+    start_hyperopt_list(pargs)
-            "--no-details",
+    captured = capsys.readouterr()
-            "--no-color",
+    assert all(x in captured.out
-            "--export-csv",
+               for x in [" 2/12", " 6/12"])
-            str(csv_file),
+    assert all(x not in captured.out
-        ]
+               for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 7/12", " 8/12"
-        pargs = get_args(args)
+                         " 9/12", " 10/12", " 11/12", " 12/12"])
-        pargs['config'] = None
+    args = [
-        start_hyperopt_list(pargs)
+        "hyperopt-list",
-        captured = capsys.readouterr()
+        "--no-details",
-        log_has("CSV file created: test_file.csv", caplog)
+        "--no-color",
-        assert csv_file.is_file()
+        "--export-csv",
-        line = csv_file.read_text()
+        str(csv_file),
-        assert ('Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,"3,930.0 m",0.43662' in line
+    ]
-                or "Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,2 days 17:30:00,0.43662" in line)
+    pargs = get_args(args)
-        csv_file.unlink()
+    pargs['config'] = None
+    start_hyperopt_list(pargs)
+    captured = capsys.readouterr()
+    log_has("CSV file created: test_file.csv", caplog)
+    assert csv_file.is_file()
+    line = csv_file.read_text()
+    assert ('Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,"3,930.0 m",0.43662' in line
+            or "Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,2 days 17:30:00,0.43662" in line)
+    csv_file.unlink()
 
 
 def test_hyperopt_show(mocker, capsys, saved_hyperopt_results):
     mocker.patch(
-        'freqtrade.optimize.hyperopt_tools.HyperoptTools.load_previous_results',
+        'freqtrade.optimize.hyperopt_tools.HyperoptTools._test_hyperopt_results_exist',
-        MagicMock(return_value=saved_hyperopt_results)
+        return_value=True
+    )
+
+    def fake_iterator(*args, **kwargs):
+        yield from [saved_hyperopt_results]
+
+    mocker.patch(
+        'freqtrade.optimize.hyperopt_tools.HyperoptTools._read_results',
+        side_effect=fake_iterator
     )
     mocker.patch('freqtrade.commands.hyperopt_commands.show_backtest_result')
 

tests/config_test_comments.json

@@ -6,8 +6,8 @@
      */
     "stake_currency": "BTC",
     "stake_amount": 0.05,
-    "fiat_display_currency": "USD",             // C++-style comment
+    "fiat_display_currency": "USD", // C++-style comment
-    "amount_reserve_percent" : 0.05,		// And more, tabs before this comment
+    "amount_reserve_percent": 0.05, // And more, tabs before this comment
     "dry_run": false,
     "timeframe": "5m",
     "trailing_stop": false,
@@ -15,15 +15,15 @@
     "trailing_stop_positive_offset": 0.0051,
     "trailing_only_offset_is_reached": false,
     "minimal_roi": {
-        "40":  0.0,
+        "40": 0.0,
-        "30":  0.01,
+        "30": 0.01,
-        "20":  0.02,
+        "20": 0.02,
-        "0":  0.04
+        "0": 0.04
     },
     "stoploss": -0.10,
     "unfilledtimeout": {
         "buy": 10,
-        "sell": 30,				// Trailing comma should also be accepted now
+        "sell": 30, // Trailing comma should also be accepted now
     },
     "bid_strategy": {
         "use_order_book": false,
@@ -34,7 +34,7 @@
             "bids_to_ask_delta": 1
         }
     },
-    "ask_strategy":{
+    "ask_strategy": {
         "use_order_book": false,
         "order_book_min": 1,
         "order_book_max": 9
@@ -64,7 +64,9 @@
         "key": "your_exchange_key",
         "secret": "your_exchange_secret",
         "password": "",
-        "ccxt_config": {"enableRateLimit": true},
+        "ccxt_config": {
+            "enableRateLimit": true
+        },
         "ccxt_async_config": {
             "enableRateLimit": false,
             "rateLimit": 500,
@@ -103,8 +105,8 @@
         "remove_pumps": false
     },
     "telegram": {
-// We can now comment out some settings
+        // We can now comment out some settings
-//        "enabled": true,
+        //        "enabled": true,
         "enabled": false,
         "token": "your_telegram_token",
         "chat_id": "your_telegram_chat_id"
@@ -124,4 +126,4 @@
     },
     "strategy": "DefaultStrategy",
     "strategy_path": "user_data/strategies/"
-}
+}

tests/conftest.py

@@ -182,7 +182,7 @@ def get_patched_worker(mocker, config) -> Worker:
     return Worker(args=None, config=config)
 
 
-def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None:
+def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False, None)) -> None:
     """
     :param mocker: mocker to patch IStrategy class
     :param value: which value IStrategy.get_signal() must return
@@ -323,7 +323,7 @@ def get_default_conf(testdatadir):
         "user_data_dir": Path("user_data"),
         "verbosity": 3,
         "strategy_path": str(Path(__file__).parent / "strategy" / "strats"),
-        "strategy": "DefaultStrategy",
+        "strategy": "StrategyTestV2",
         "disableparamexport": True,
         "internals": {},
         "export": "none",
@@ -812,7 +812,7 @@ def shitcoinmarkets(markets):
             "future": False,
             "active": True
         },
-        })
+    })
     return shitmarkets
 
 
@@ -1115,7 +1115,7 @@ def order_book_l2_usd():
             [25.576, 262.016],
             [25.577, 178.557],
             [25.578, 78.614]
-            ],
+        ],
         'timestamp': None,
         'datetime': None,
         'nonce': 2372149736
@@ -1814,138 +1814,6 @@ def open_trade():
     )
 
 
-@pytest.fixture
-def saved_hyperopt_results_legacy():
-    return [
-        {
-            'loss': 0.4366182531161519,
-            'params_dict': {
-                'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1190, 'roi_t2': 541, 'roi_t3': 408, 'roi_p1': 0.026035863879169705, 'roi_p2': 0.12508730043628782, 'roi_p3': 0.27766427921605896, 'stoploss': -0.2562930402099556},  # noqa: E501
-            'params_details': {'buy': {'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.4287874435315165, 408: 0.15112316431545753, 949: 0.026035863879169705, 2139: 0}, 'stoploss': {'stoploss': -0.2562930402099556}},  # noqa: E501
-            'results_metrics': {'trade_count': 2, 'avg_profit': -1.254995, 'median_profit': -1.2222, 'total_profit': -0.00125625, 'profit': -2.50999, 'duration': 3930.0},  # noqa: E501
-            'results_explanation': '     2 trades. Avg profit  -1.25%. Total profit -0.00125625 BTC (  -2.51Σ%). Avg duration 3930.0 min.',  # noqa: E501
-            'total_profit': -0.00125625,
-            'current_epoch': 1,
-            'is_initial_point': True,
-            'is_best': True
-        }, {
-            'loss': 20.0,
-            'params_dict': {
-                'mfi-value': 17, 'fastd-value': 38, 'adx-value': 48, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 96, 'sell-fastd-value': 68, 'sell-adx-value': 63, 'sell-rsi-value': 81, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal', 'roi_t1': 334, 'roi_t2': 683, 'roi_t3': 140, 'roi_p1': 0.06403981740598495, 'roi_p2': 0.055519840060645045, 'roi_p3': 0.3253712811342459, 'stoploss': -0.338070047333259},  # noqa: E501
-            'params_details': {
-                'buy': {'mfi-value': 17, 'fastd-value': 38, 'adx-value': 48, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'},  # noqa: E501
-                'sell': {'sell-mfi-value': 96, 'sell-fastd-value': 68, 'sell-adx-value': 63, 'sell-rsi-value': 81, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal'},  # noqa: E501
-                'roi': {0: 0.4449309386008759, 140: 0.11955965746663, 823: 0.06403981740598495, 1157: 0},  # noqa: E501
-                'stoploss': {'stoploss': -0.338070047333259}},
-            'results_metrics': {'trade_count': 1, 'avg_profit': 0.12357, 'median_profit': -1.2222, 'total_profit': 6.185e-05, 'profit': 0.12357, 'duration': 1200.0},  # noqa: E501
-            'results_explanation': '     1 trades. Avg profit   0.12%. Total profit  0.00006185 BTC (   0.12Σ%). Avg duration 1200.0 min.',  # noqa: E501
-            'total_profit': 6.185e-05,
-            'current_epoch': 2,
-            'is_initial_point': True,
-            'is_best': False
-        }, {
-            'loss': 14.241196856510731,
-            'params_dict': {'mfi-value': 25, 'fastd-value': 16, 'adx-value': 29, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 98, 'sell-fastd-value': 72, 'sell-adx-value': 51, 'sell-rsi-value': 82, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 889, 'roi_t2': 533, 'roi_t3': 263, 'roi_p1': 0.04759065393663096, 'roi_p2': 0.1488819964638463, 'roi_p3': 0.4102801822104605, 'stoploss': -0.05394588767607611},  # noqa: E501
-            'params_details': {'buy': {'mfi-value': 25, 'fastd-value': 16, 'adx-value': 29, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 98, 'sell-fastd-value': 72, 'sell-adx-value': 51, 'sell-rsi-value': 82, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.6067528326109377, 263: 0.19647265040047726, 796: 0.04759065393663096, 1685: 0}, 'stoploss': {'stoploss': -0.05394588767607611}},  # noqa: E501
-            'results_metrics': {'trade_count': 621, 'avg_profit': -0.43883302093397747, 'median_profit': -1.2222, 'total_profit': -0.13639474, 'profit': -272.515306, 'duration': 1691.207729468599},  # noqa: E501
-            'results_explanation': '   621 trades. Avg profit  -0.44%. Total profit -0.13639474 BTC (-272.52Σ%). Avg duration 1691.2 min.',  # noqa: E501
-            'total_profit': -0.13639474,
-            'current_epoch': 3,
-            'is_initial_point': True,
-            'is_best': False
-        }, {
-            'loss': 100000,
-            'params_dict': {'mfi-value': 13, 'fastd-value': 35, 'adx-value': 39, 'rsi-value': 29, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 87, 'sell-fastd-value': 54, 'sell-adx-value': 63, 'sell-rsi-value': 93, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1402, 'roi_t2': 676, 'roi_t3': 215, 'roi_p1': 0.06264755784937427, 'roi_p2': 0.14258587851894644, 'roi_p3': 0.20671291201040828, 'stoploss': -0.11818343570194478},  # noqa: E501
-            'params_details': {'buy': {'mfi-value': 13, 'fastd-value': 35, 'adx-value': 39, 'rsi-value': 29, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 54, 'sell-adx-value': 63, 'sell-rsi-value': 93, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.411946348378729, 215: 0.2052334363683207, 891: 0.06264755784937427, 2293: 0}, 'stoploss': {'stoploss': -0.11818343570194478}},  # noqa: E501
-            'results_metrics': {'trade_count': 0, 'avg_profit': None, 'median_profit': None, 'total_profit': 0, 'profit': 0.0, 'duration': None},  # noqa: E501
-            'results_explanation': '     0 trades. Avg profit    nan%. Total profit  0.00000000 BTC (   0.00Σ%). Avg duration   nan min.',  # noqa: E501
-            'total_profit': 0, 'current_epoch': 4, 'is_initial_point': True, 'is_best': False
-        }, {
-            'loss': 0.22195522184191518,
-            'params_dict': {'mfi-value': 17, 'fastd-value': 21, 'adx-value': 38, 'rsi-value': 33, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 87, 'sell-fastd-value': 82, 'sell-adx-value': 78, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 1269, 'roi_t2': 601, 'roi_t3': 444, 'roi_p1': 0.07280999507931168, 'roi_p2': 0.08946698095898986, 'roi_p3': 0.1454876733325284, 'stoploss': -0.18181041180901014},   # noqa: E501
-            'params_details': {'buy': {'mfi-value': 17, 'fastd-value': 21, 'adx-value': 38, 'rsi-value': 33, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 82, 'sell-adx-value': 78, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.3077646493708299, 444: 0.16227697603830155, 1045: 0.07280999507931168, 2314: 0}, 'stoploss': {'stoploss': -0.18181041180901014}},  # noqa: E501
-            'results_metrics': {'trade_count': 14, 'avg_profit': -0.3539515, 'median_profit': -1.2222, 'total_profit': -0.002480140000000001, 'profit': -4.955321, 'duration': 3402.8571428571427},  # noqa: E501
-            'results_explanation': '    14 trades. Avg profit  -0.35%. Total profit -0.00248014 BTC (  -4.96Σ%). Avg duration 3402.9 min.',  # noqa: E501
-            'total_profit': -0.002480140000000001,
-            'current_epoch': 5,
-            'is_initial_point': True,
-            'is_best': True
-        }, {
-            'loss': 0.545315889154162,
-            'params_dict': {'mfi-value': 22, 'fastd-value': 43, 'adx-value': 46, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'bb_lower', 'sell-mfi-value': 87, 'sell-fastd-value': 65, 'sell-adx-value': 94, 'sell-rsi-value': 63, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 319, 'roi_t2': 556, 'roi_t3': 216, 'roi_p1': 0.06251955472249589, 'roi_p2': 0.11659519602202795, 'roi_p3': 0.0953744132197762, 'stoploss': -0.024551752215582423},  # noqa: E501
-            'params_details': {'buy': {'mfi-value': 22, 'fastd-value': 43, 'adx-value': 46, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'bb_lower'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 65, 'sell-adx-value': 94, 'sell-rsi-value': 63, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.2744891639643, 216: 0.17911475074452382, 772: 0.06251955472249589, 1091: 0}, 'stoploss': {'stoploss': -0.024551752215582423}},  # noqa: E501
-            'results_metrics': {'trade_count': 39, 'avg_profit': -0.21400679487179478, 'median_profit': -1.2222, 'total_profit': -0.0041773, 'profit': -8.346264999999997, 'duration': 636.9230769230769},  # noqa: E501
-            'results_explanation': '    39 trades. Avg profit  -0.21%. Total profit -0.00417730 BTC (  -8.35Σ%). Avg duration 636.9 min.',  # noqa: E501
-            'total_profit': -0.0041773,
-            'current_epoch': 6,
-            'is_initial_point': True,
-            'is_best': False
-        }, {
-            'loss': 4.713497421432944,
-            'params_dict': {'mfi-value': 13, 'fastd-value': 41, 'adx-value': 21, 'rsi-value': 29, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower', 'sell-mfi-value': 99, 'sell-fastd-value': 60, 'sell-adx-value': 81, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 771, 'roi_t2': 620, 'roi_t3': 145, 'roi_p1': 0.0586919200378493, 'roi_p2': 0.04984118697312542, 'roi_p3': 0.37521058680247044, 'stoploss': -0.14613268022709905},  # noqa: E501
-            'params_details': {
-                'buy': {'mfi-value': 13, 'fastd-value': 41, 'adx-value': 21, 'rsi-value': 29, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower'}, 'sell': {'sell-mfi-value': 99, 'sell-fastd-value': 60, 'sell-adx-value': 81, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.4837436938134452, 145: 0.10853310701097472, 765: 0.0586919200378493, 1536: 0},  # noqa: E501
-                'stoploss': {'stoploss': -0.14613268022709905}},  # noqa: E501
-            'results_metrics': {'trade_count': 318, 'avg_profit': -0.39833954716981146, 'median_profit': -1.2222, 'total_profit': -0.06339929, 'profit': -126.67197600000004, 'duration': 3140.377358490566},  # noqa: E501
-            'results_explanation': '   318 trades. Avg profit  -0.40%. Total profit -0.06339929 BTC (-126.67Σ%). Avg duration 3140.4 min.',  # noqa: E501
-            'total_profit': -0.06339929,
-            'current_epoch': 7,
-            'is_initial_point': True,
-            'is_best': False
-        }, {
-            'loss': 20.0,  # noqa: E501
-            'params_dict': {'mfi-value': 24, 'fastd-value': 43, 'adx-value': 33, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'sar_reversal', 'sell-mfi-value': 89, 'sell-fastd-value': 74, 'sell-adx-value': 70, 'sell-rsi-value': 70, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': False, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal', 'roi_t1': 1149, 'roi_t2': 375, 'roi_t3': 289, 'roi_p1': 0.05571820757172588, 'roi_p2': 0.0606240398618907, 'roi_p3': 0.1729012220156157, 'stoploss': -0.1588514289110401},  # noqa: E501
-            'params_details': {'buy': {'mfi-value': 24, 'fastd-value': 43, 'adx-value': 33, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'sar_reversal'}, 'sell': {'sell-mfi-value': 89, 'sell-fastd-value': 74, 'sell-adx-value': 70, 'sell-rsi-value': 70, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': False, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal'}, 'roi': {0: 0.2892434694492323, 289: 0.11634224743361658, 664: 0.05571820757172588, 1813: 0}, 'stoploss': {'stoploss': -0.1588514289110401}},  # noqa: E501
-            'results_metrics': {'trade_count': 1, 'avg_profit': 0.0, 'median_profit': 0.0, 'total_profit': 0.0, 'profit': 0.0, 'duration': 5340.0},  # noqa: E501
-            'results_explanation': '     1 trades. Avg profit   0.00%. Total profit  0.00000000 BTC (   0.00Σ%). Avg duration 5340.0 min.',  # noqa: E501
-            'total_profit': 0.0,
-            'current_epoch': 8,
-            'is_initial_point': True,
-            'is_best': False
-        }, {
-            'loss': 2.4731817780991223,
-            'params_dict': {'mfi-value': 22, 'fastd-value': 20, 'adx-value': 29, 'rsi-value': 40, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'sar_reversal', 'sell-mfi-value': 97, 'sell-fastd-value': 65, 'sell-adx-value': 81, 'sell-rsi-value': 64, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1012, 'roi_t2': 584, 'roi_t3': 422, 'roi_p1': 0.036764323603472565, 'roi_p2': 0.10335480573205287, 'roi_p3': 0.10322347377503042, 'stoploss': -0.2780610808108503},  # noqa: E501
-            'params_details': {'buy': {'mfi-value': 22, 'fastd-value': 20, 'adx-value': 29, 'rsi-value': 40, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'sar_reversal'}, 'sell': {'sell-mfi-value': 97, 'sell-fastd-value': 65, 'sell-adx-value': 81, 'sell-rsi-value': 64, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.2433426031105559, 422: 0.14011912933552545, 1006: 0.036764323603472565, 2018: 0}, 'stoploss': {'stoploss': -0.2780610808108503}},  # noqa: E501
-            'results_metrics': {'trade_count': 229, 'avg_profit': -0.38433433624454144, 'median_profit': -1.2222, 'total_profit': -0.044050070000000004, 'profit': -88.01256299999999, 'duration': 6505.676855895196},  # noqa: E501
-            'results_explanation': '   229 trades. Avg profit  -0.38%. Total profit -0.04405007 BTC ( -88.01Σ%). Avg duration 6505.7 min.',  # noqa: E501
-            'total_profit': -0.044050070000000004,  # noqa: E501
-            'current_epoch': 9,
-            'is_initial_point': True,
-            'is_best': False
-        }, {
-            'loss': -0.2604606005845212,  # noqa: E501
-            'params_dict': {'mfi-value': 23, 'fastd-value': 24, 'adx-value': 22, 'rsi-value': 24, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 97, 'sell-fastd-value': 70, 'sell-adx-value': 64, 'sell-rsi-value': 80, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal', 'roi_t1': 792, 'roi_t2': 464, 'roi_t3': 215, 'roi_p1': 0.04594053535385903, 'roi_p2': 0.09623192684243963, 'roi_p3': 0.04428219070850663, 'stoploss': -0.16992287161634415},  # noqa: E501
-            'params_details': {'buy': {'mfi-value': 23, 'fastd-value': 24, 'adx-value': 22, 'rsi-value': 24, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 97, 'sell-fastd-value': 70, 'sell-adx-value': 64, 'sell-rsi-value': 80, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal'}, 'roi': {0: 0.18645465290480528, 215: 0.14217246219629864, 679: 0.04594053535385903, 1471: 0}, 'stoploss': {'stoploss': -0.16992287161634415}},  # noqa: E501
-            'results_metrics': {'trade_count': 4, 'avg_profit': 0.1080385, 'median_profit': -1.2222, 'total_profit': 0.00021629, 'profit': 0.432154, 'duration': 2850.0},  # noqa: E501
-            'results_explanation': '     4 trades. Avg profit   0.11%. Total profit  0.00021629 BTC (   0.43Σ%). Avg duration 2850.0 min.',  # noqa: E501
-            'total_profit': 0.00021629,
-            'current_epoch': 10,
-            'is_initial_point': True,
-            'is_best': True
-        }, {
-            'loss': 4.876465945994304,  # noqa: E501
-            'params_dict': {'mfi-value': 20, 'fastd-value': 32, 'adx-value': 49, 'rsi-value': 23, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower', 'sell-mfi-value': 75, 'sell-fastd-value': 56, 'sell-adx-value': 61, 'sell-rsi-value': 62, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 579, 'roi_t2': 614, 'roi_t3': 273, 'roi_p1': 0.05307643172744114, 'roi_p2': 0.1352282078262871, 'roi_p3': 0.1913307406325751, 'stoploss': -0.25728526022513887},  # noqa: E501
-            'params_details': {'buy': {'mfi-value': 20, 'fastd-value': 32, 'adx-value': 49, 'rsi-value': 23, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower'}, 'sell': {'sell-mfi-value': 75, 'sell-fastd-value': 56, 'sell-adx-value': 61, 'sell-rsi-value': 62, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.3796353801863034, 273: 0.18830463955372825, 887: 0.05307643172744114, 1466: 0}, 'stoploss': {'stoploss': -0.25728526022513887}},  # noqa: E501
-            'results_metrics': {'trade_count': 117, 'avg_profit': -1.2698609145299145, 'median_profit': -1.2222, 'total_profit': -0.07436117, 'profit': -148.573727, 'duration': 4282.5641025641025},  # noqa: E501
-            'results_explanation': '   117 trades. Avg profit  -1.27%. Total profit -0.07436117 BTC (-148.57Σ%). Avg duration 4282.6 min.',  # noqa: E501
-            'total_profit': -0.07436117,
-            'current_epoch': 11,
-            'is_initial_point': True,
-            'is_best': False
-        }, {
-            'loss': 100000,
-            'params_dict': {'mfi-value': 10, 'fastd-value': 36, 'adx-value': 31, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'sar_reversal', 'sell-mfi-value': 80, 'sell-fastd-value': 71, 'sell-adx-value': 60, 'sell-rsi-value': 85, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1156, 'roi_t2': 581, 'roi_t3': 408, 'roi_p1': 0.06860454019988212, 'roi_p2': 0.12473718444931989, 'roi_p3': 0.2896360635226823, 'stoploss': -0.30889015124682806},  # noqa: E501
-            'params_details': {'buy': {'mfi-value': 10, 'fastd-value': 36, 'adx-value': 31, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'sar_reversal'}, 'sell': {'sell-mfi-value': 80, 'sell-fastd-value': 71, 'sell-adx-value': 60, 'sell-rsi-value': 85, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.4829777881718843, 408: 0.19334172464920202, 989: 0.06860454019988212, 2145: 0}, 'stoploss': {'stoploss': -0.30889015124682806}},  # noqa: E501
-            'results_metrics': {'trade_count': 0, 'avg_profit': None, 'median_profit': None, 'total_profit': 0, 'profit': 0.0, 'duration': None},  # noqa: E501
-            'results_explanation': '     0 trades. Avg profit    nan%. Total profit  0.00000000 BTC (   0.00Σ%). Avg duration   nan min.',  # noqa: E501
-            'total_profit': 0,
-            'current_epoch': 12,
-            'is_initial_point': True,
-            'is_best': False
-            }
-    ]
-
-
 @pytest.fixture
 def saved_hyperopt_results():
     hyperopt_res = [
@@ -2084,3 +1952,88 @@ def saved_hyperopt_results():
                                                                          ].total_seconds()
 
     return hyperopt_res
+
+
+@pytest.fixture(scope='function')
+def limit_buy_order_usdt_open():
+    return {
+        'id': 'mocked_limit_buy',
+        'type': 'limit',
+        'side': 'buy',
+        'symbol': 'mocked',
+        'datetime': arrow.utcnow().isoformat(),
+        'timestamp': arrow.utcnow().int_timestamp,
+        'price': 2.00,
+        'amount': 30.0,
+        'filled': 0.0,
+        'cost': 60.0,
+        'remaining': 30.0,
+        'status': 'open'
+    }
+
+
+@pytest.fixture(scope='function')
+def limit_buy_order_usdt(limit_buy_order_usdt_open):
+    order = deepcopy(limit_buy_order_usdt_open)
+    order['status'] = 'closed'
+    order['filled'] = order['amount']
+    order['remaining'] = 0.0
+    return order
+
+
+@pytest.fixture
+def limit_sell_order_usdt_open():
+    return {
+        'id': 'mocked_limit_sell',
+        'type': 'limit',
+        'side': 'sell',
+        'pair': 'mocked',
+        'datetime': arrow.utcnow().isoformat(),
+        'timestamp': arrow.utcnow().int_timestamp,
+        'price': 2.20,
+        'amount': 30.0,
+        'filled': 0.0,
+        'remaining': 30.0,
+        'status': 'open'
+    }
+
+
+@pytest.fixture
+def limit_sell_order_usdt(limit_sell_order_usdt_open):
+    order = deepcopy(limit_sell_order_usdt_open)
+    order['remaining'] = 0.0
+    order['filled'] = order['amount']
+    order['status'] = 'closed'
+    return order
+
+
+@pytest.fixture(scope='function')
+def market_buy_order_usdt():
+    return {
+        'id': 'mocked_market_buy',
+        'type': 'market',
+        'side': 'buy',
+        'symbol': 'mocked',
+        'datetime': arrow.utcnow().isoformat(),
+        'price': 2.00,
+        'amount': 30.0,
+        'filled': 30.0,
+        'remaining': 0.0,
+        'status': 'closed'
+    }
+
+
+@pytest.fixture
+def market_sell_order_usdt():
+    return {
+        'id': 'mocked_limit_sell',
+        'type': 'market',
+        'side': 'sell',
+        'symbol': 'mocked',
+        'datetime': arrow.utcnow().isoformat(),
+        'price': 2.20,
+        'amount': 30.0,
+        'filled': 30.0,
+        'remaining': 0.0,
+        'status': 'closed'
+    }

tests/conftest_trades.py

@@ -33,7 +33,7 @@ def mock_trade_1(fee):
         open_rate=0.123,
         exchange='binance',
         open_order_id='dry_run_buy_12345',
-        strategy='DefaultStrategy',
+        strategy='StrategyTestV2',
         timeframe=5,
     )
     o = Order.parse_from_ccxt_object(mock_order_1(), 'ETH/BTC', 'buy')
@@ -87,7 +87,7 @@ def mock_trade_2(fee):
         exchange='binance',
         is_open=False,
         open_order_id='dry_run_sell_12345',
-        strategy='DefaultStrategy',
+        strategy='StrategyTestV2',
         timeframe=5,
         sell_reason='sell_signal',
         open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20),
@@ -146,7 +146,7 @@ def mock_trade_3(fee):
         close_profit_abs=0.000155,
         exchange='binance',
         is_open=False,
-        strategy='DefaultStrategy',
+        strategy='StrategyTestV2',
         timeframe=5,
         sell_reason='roi',
         open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20),
@@ -189,7 +189,7 @@ def mock_trade_4(fee):
         open_rate=0.123,
         exchange='binance',
         open_order_id='prod_buy_12345',
-        strategy='DefaultStrategy',
+        strategy='StrategyTestV2',
         timeframe=5,
     )
     o = Order.parse_from_ccxt_object(mock_order_4(), 'ETC/BTC', 'buy')

tests/data/test_btanalysis.py

@@ -93,7 +93,7 @@ def test_load_backtest_data_new_format(testdatadir):
 def test_load_backtest_data_multi(testdatadir):
 
     filename = testdatadir / "backtest-result_multistrat.json"
-    for strategy in ('DefaultStrategy', 'TestStrategy'):
+    for strategy in ('StrategyTestV2', 'TestStrategy'):
         bt_data = load_backtest_data(filename, strategy=strategy)
         assert isinstance(bt_data, DataFrame)
         assert set(bt_data.columns) == set(BT_DATA_COLUMNS_MID)
@@ -128,7 +128,7 @@ def test_load_trades_from_db(default_conf, fee, mocker):
     for col in BT_DATA_COLUMNS:
         if col not in ['index', 'open_at_end']:
             assert col in trades.columns
-    trades = load_trades_from_db(db_url=default_conf['db_url'], strategy='DefaultStrategy')
+    trades = load_trades_from_db(db_url=default_conf['db_url'], strategy='StrategyTestV2')
     assert len(trades) == 4
     trades = load_trades_from_db(db_url=default_conf['db_url'], strategy='NoneStrategy')
     assert len(trades) == 0
@@ -186,7 +186,7 @@ def test_load_trades(default_conf, mocker):
                 db_url=default_conf.get('db_url'),
                 exportfilename=default_conf.get('exportfilename'),
                 no_trades=False,
-                strategy="DefaultStrategy",
+                strategy="StrategyTestV2",
                 )
 
     assert db_mock.call_count == 1

tests/data/test_converter.py

@@ -119,7 +119,7 @@ def test_ohlcv_fill_up_missing_data2(caplog):
     # 3rd candle has been filled
     row = data2.loc[2, :]
     assert row['volume'] == 0
-    # close shoult match close of previous candle
+    # close should match close of previous candle
     assert row['close'] == data.loc[1, 'close']
     assert row['open'] == row['close']
     assert row['high'] == row['close']

tests/data/test_dataprovider.py

@@ -66,7 +66,7 @@ def test_historic_ohlcv_dataformat(mocker, default_conf, ohlcv_history):
     hdf5loadmock.assert_not_called()
     jsonloadmock.assert_called_once()
 
-    # Swiching to dataformat hdf5
+    # Switching to dataformat hdf5
     hdf5loadmock.reset_mock()
     jsonloadmock.reset_mock()
     default_conf["dataformat_ohlcv"] = "hdf5"

tests/data/test_history.py

@@ -133,8 +133,8 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog,
     load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC')
     assert file.is_file()
     assert log_has_re(
-        'Download history data for pair: "MEME/BTC", timeframe: 1m '
+        r'Download history data for pair: "MEME/BTC" \(0/1\), timeframe: 1m '
-        'and store in .*', caplog
+        r'and store in .*', caplog
     )
 
 
@@ -200,15 +200,15 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None:
     assert start_ts == test_data[0][0] - 1000
 
     # timeframe starts in the center of the cached data
-    # should return the chached data w/o the last item
+    # should return the cached data w/o the last item
     timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0)
     data, start_ts = _load_cached_data_for_updating('UNITTEST/BTC', '1m', timerange, data_handler)
 
     assert_frame_equal(data, test_data_df.iloc[:-1])
     assert test_data[-2][0] <= start_ts < test_data[-1][0]
 
-    # timeframe starts after the chached data
+    # timeframe starts after the cached data
-    # should return the chached data w/o the last item
+    # should return the cached data w/o the last item
     timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 100, 0)
     data, start_ts = _load_cached_data_for_updating('UNITTEST/BTC', '1m', timerange, data_handler)
     assert_frame_equal(data, test_data_df.iloc[:-1])
@@ -278,8 +278,10 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None:
         return_value=None)
     mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick)
     exchange = get_patched_exchange(mocker, default_conf)
-    _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m')
+    _download_pair_history(datadir=testdatadir, exchange=exchange, pair="UNITTEST/BTC",
-    _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m')
+                           timeframe='1m')
+    _download_pair_history(datadir=testdatadir, exchange=exchange, pair="UNITTEST/BTC",
+                           timeframe='3m')
     assert json_dump_mock.call_count == 2
 
 
@@ -378,10 +380,10 @@ def test_file_dump_json_tofile(testdatadir) -> None:
 def test_get_timerange(default_conf, mocker, testdatadir) -> None:
     patch_exchange(mocker)
 
-    default_conf.update({'strategy': 'DefaultStrategy'})
+    default_conf.update({'strategy': 'StrategyTestV2'})
     strategy = StrategyResolver.load_strategy(default_conf)
 
-    data = strategy.ohlcvdata_to_dataframe(
+    data = strategy.advise_all_indicators(
         load_data(
             datadir=testdatadir,
             timeframe='1m',
@@ -396,10 +398,10 @@ def test_get_timerange(default_conf, mocker, testdatadir) -> None:
 def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir) -> None:
     patch_exchange(mocker)
 
-    default_conf.update({'strategy': 'DefaultStrategy'})
+    default_conf.update({'strategy': 'StrategyTestV2'})
     strategy = StrategyResolver.load_strategy(default_conf)
 
-    data = strategy.ohlcvdata_to_dataframe(
+    data = strategy.advise_all_indicators(
         load_data(
             datadir=testdatadir,
             timeframe='1m',
@@ -420,11 +422,11 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir)
 def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> None:
     patch_exchange(mocker)
 
-    default_conf.update({'strategy': 'DefaultStrategy'})
+    default_conf.update({'strategy': 'StrategyTestV2'})
     strategy = StrategyResolver.load_strategy(default_conf)
 
     timerange = TimeRange('index', 'index', 200, 250)
-    data = strategy.ohlcvdata_to_dataframe(
+    data = strategy.advise_all_indicators(
         load_data(
             datadir=testdatadir,
             timeframe='5m',

tests/edge/test_edge.py

@@ -29,7 +29,6 @@ from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe,
 
 tests_start_time = arrow.get(2018, 10, 3)
 timeframe_in_minute = 60
-_ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7}
 
 # Helpers for this test file
 

tests/exchange/test_ccxt_compat.py

@@ -42,6 +42,11 @@ EXCHANGES = {
         'hasQuoteVolume': True,
         'timeframe': '5m',
     },
+    'gateio': {
+        'pair': 'BTC/USDT',
+        'hasQuoteVolume': True,
+        'timeframe': '5m',
+    },
 }
 
 
@@ -142,8 +147,8 @@ class TestCCXTExchange():
     def test_ccxt_get_fee(self, exchange):
         exchange, exchangename = exchange
         pair = EXCHANGES[exchangename]['pair']
-
+        threshold = 0.01
-        assert 0 < exchange.get_fee(pair, 'limit', 'buy') < 1
+        assert 0 < exchange.get_fee(pair, 'limit', 'buy') < threshold
-        assert 0 < exchange.get_fee(pair, 'limit', 'sell') < 1
+        assert 0 < exchange.get_fee(pair, 'limit', 'sell') < threshold
-        assert 0 < exchange.get_fee(pair, 'market', 'buy') < 1
+        assert 0 < exchange.get_fee(pair, 'market', 'buy') < threshold
-        assert 0 < exchange.get_fee(pair, 'market', 'sell') < 1
+        assert 0 < exchange.get_fee(pair, 'market', 'sell') < threshold

tests/exchange/test_exchange.py

@@ -984,16 +984,21 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice,
     assert order['fee']
 
 
-@pytest.mark.parametrize("side,amount,endprice", [
+@pytest.mark.parametrize("side,rate,amount,endprice", [
-    ("buy", 1, 25.566),
+    # spread is 25.263-25.266
-    ("buy", 100, 25.5672),  # Requires interpolation
+    ("buy", 25.564, 1, 25.566),
-    ("buy", 1000, 25.575),  # More than orderbook return
+    ("buy", 25.564, 100, 25.5672),  # Requires interpolation
-    ("sell", 1, 25.563),
+    ("buy", 25.590, 100, 25.5672),  # Price above spread ... average is lower
-    ("sell", 100, 25.5625),  # Requires interpolation
+    ("buy", 25.564, 1000, 25.575),  # More than orderbook return
-    ("sell", 1000, 25.5555),  # More than orderbook return
+    ("buy", 24.000, 100000, 25.200),  # Run into max_slippage of 5%
+    ("sell", 25.564, 1, 25.563),
+    ("sell", 25.564, 100, 25.5625),  # Requires interpolation
+    ("sell", 25.510, 100, 25.5625),  # price below spread - average is higher
+    ("sell", 25.564, 1000, 25.5555),  # More than orderbook return
+    ("sell", 27, 10000, 25.65),  # max-slippage 5%
 ])
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
-def test_create_dry_run_order_market_fill(default_conf, mocker, side, amount, endprice,
+def test_create_dry_run_order_market_fill(default_conf, mocker, side, rate, amount, endprice,
                                           exchange_name, order_book_l2_usd):
     default_conf['dry_run'] = True
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
@@ -1003,7 +1008,7 @@ def test_create_dry_run_order_market_fill(default_conf, mocker, side, amount, en
                           )
 
     order = exchange.create_dry_run_order(
-        pair='LTC/USDT', ordertype='market', side=side, amount=amount, rate=25.5)
+        pair='LTC/USDT', ordertype='market', side=side, amount=amount, rate=rate)
     assert 'id' in order
     assert f'dry_run_{side}_' in order["id"]
     assert order["side"] == side
@@ -1056,8 +1061,8 @@ def test_buy_dry_run(default_conf, mocker):
     default_conf['dry_run'] = True
     exchange = get_patched_exchange(mocker, default_conf)
 
-    order = exchange.buy(pair='ETH/BTC', ordertype='limit',
+    order = exchange.create_order(pair='ETH/BTC', ordertype='limit', side="buy",
-                         amount=1, rate=200, time_in_force='gtc')
+                                  amount=1, rate=200, time_in_force='gtc')
     assert 'id' in order
     assert 'dry_run_buy_' in order['id']
 
@@ -1080,8 +1085,8 @@ def test_buy_prod(default_conf, mocker, exchange_name):
     mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
 
-    order = exchange.buy(pair='ETH/BTC', ordertype=order_type,
+    order = exchange.create_order(pair='ETH/BTC', ordertype=order_type, side="buy",
-                         amount=1, rate=200, time_in_force=time_in_force)
+                                  amount=1, rate=200, time_in_force=time_in_force)
 
     assert 'id' in order
     assert 'info' in order
@@ -1094,9 +1099,10 @@ def test_buy_prod(default_conf, mocker, exchange_name):
 
     api_mock.create_order.reset_mock()
     order_type = 'limit'
-    order = exchange.buy(
+    order = exchange.create_order(
         pair='ETH/BTC',
         ordertype=order_type,
+        side="buy",
         amount=1,
         rate=200,
         time_in_force=time_in_force)
@@ -1110,32 +1116,32 @@ def test_buy_prod(default_conf, mocker, exchange_name):
     with pytest.raises(DependencyException):
         api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("Not enough funds"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-        exchange.buy(pair='ETH/BTC', ordertype=order_type,
+        exchange.create_order(pair='ETH/BTC', ordertype=order_type, side="buy",
-                     amount=1, rate=200, time_in_force=time_in_force)
+                              amount=1, rate=200, time_in_force=time_in_force)
 
     with pytest.raises(DependencyException):
         api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-        exchange.buy(pair='ETH/BTC', ordertype='limit',
+        exchange.create_order(pair='ETH/BTC', ordertype='limit', side="buy",
-                     amount=1, rate=200, time_in_force=time_in_force)
+                              amount=1, rate=200, time_in_force=time_in_force)
 
     with pytest.raises(DependencyException):
         api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-        exchange.buy(pair='ETH/BTC', ordertype='market',
+        exchange.create_order(pair='ETH/BTC', ordertype='market', side="buy",
-                     amount=1, rate=200, time_in_force=time_in_force)
+                              amount=1, rate=200, time_in_force=time_in_force)
 
     with pytest.raises(TemporaryError):
         api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("Network disconnect"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-        exchange.buy(pair='ETH/BTC', ordertype=order_type,
+        exchange.create_order(pair='ETH/BTC', ordertype=order_type, side="buy",
-                     amount=1, rate=200, time_in_force=time_in_force)
+                              amount=1, rate=200, time_in_force=time_in_force)
 
     with pytest.raises(OperationalException):
         api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("Unknown error"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-        exchange.buy(pair='ETH/BTC', ordertype=order_type,
+        exchange.create_order(pair='ETH/BTC', ordertype=order_type, side="buy",
-                     amount=1, rate=200, time_in_force=time_in_force)
+                              amount=1, rate=200, time_in_force=time_in_force)
 
 
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
@@ -1157,8 +1163,8 @@ def test_buy_considers_time_in_force(default_conf, mocker, exchange_name):
     order_type = 'limit'
     time_in_force = 'ioc'
 
-    order = exchange.buy(pair='ETH/BTC', ordertype=order_type,
+    order = exchange.create_order(pair='ETH/BTC', ordertype=order_type, side="buy",
-                         amount=1, rate=200, time_in_force=time_in_force)
+                                  amount=1, rate=200, time_in_force=time_in_force)
 
     assert 'id' in order
     assert 'info' in order
@@ -1174,8 +1180,8 @@ def test_buy_considers_time_in_force(default_conf, mocker, exchange_name):
     order_type = 'market'
     time_in_force = 'ioc'
 
-    order = exchange.buy(pair='ETH/BTC', ordertype=order_type,
+    order = exchange.create_order(pair='ETH/BTC', ordertype=order_type, side="buy",
-                         amount=1, rate=200, time_in_force=time_in_force)
+                                  amount=1, rate=200, time_in_force=time_in_force)
 
     assert 'id' in order
     assert 'info' in order
@@ -1193,7 +1199,8 @@ def test_sell_dry_run(default_conf, mocker):
     default_conf['dry_run'] = True
     exchange = get_patched_exchange(mocker, default_conf)
 
-    order = exchange.sell(pair='ETH/BTC', ordertype='limit', amount=1, rate=200)
+    order = exchange.create_order(pair='ETH/BTC', ordertype='limit',
+                                  side="sell", amount=1, rate=200)
     assert 'id' in order
     assert 'dry_run_sell_' in order['id']
 
@@ -1216,7 +1223,8 @@ def test_sell_prod(default_conf, mocker, exchange_name):
     mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
 
-    order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200)
+    order = exchange.create_order(pair='ETH/BTC', ordertype=order_type,
+                                  side="sell", amount=1, rate=200)
 
     assert 'id' in order
     assert 'info' in order
@@ -1229,7 +1237,8 @@ def test_sell_prod(default_conf, mocker, exchange_name):
 
     api_mock.create_order.reset_mock()
     order_type = 'limit'
-    order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200)
+    order = exchange.create_order(pair='ETH/BTC', ordertype=order_type,
+                                  side="sell", amount=1, rate=200)
     assert api_mock.create_order.call_args[0][0] == 'ETH/BTC'
     assert api_mock.create_order.call_args[0][1] == order_type
     assert api_mock.create_order.call_args[0][2] == 'sell'
@@ -1240,28 +1249,28 @@ def test_sell_prod(default_conf, mocker, exchange_name):
     with pytest.raises(DependencyException):
         api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-        exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200)
+        exchange.create_order(pair='ETH/BTC', ordertype=order_type, side="sell", amount=1, rate=200)
 
     with pytest.raises(DependencyException):
         api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-        exchange.sell(pair='ETH/BTC', ordertype='limit', amount=1, rate=200)
+        exchange.create_order(pair='ETH/BTC', ordertype='limit', side="sell", amount=1, rate=200)
 
     # Market orders don't require price, so the behaviour is slightly different
     with pytest.raises(DependencyException):
         api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-        exchange.sell(pair='ETH/BTC', ordertype='market', amount=1, rate=200)
+        exchange.create_order(pair='ETH/BTC', ordertype='market', side="sell", amount=1, rate=200)
 
     with pytest.raises(TemporaryError):
         api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No Connection"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-        exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200)
+        exchange.create_order(pair='ETH/BTC', ordertype=order_type, side="sell", amount=1, rate=200)
 
     with pytest.raises(OperationalException):
         api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-        exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200)
+        exchange.create_order(pair='ETH/BTC', ordertype=order_type, side="sell", amount=1, rate=200)
 
 
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
@@ -1283,8 +1292,8 @@ def test_sell_considers_time_in_force(default_conf, mocker, exchange_name):
     order_type = 'limit'
     time_in_force = 'ioc'
 
-    order = exchange.sell(pair='ETH/BTC', ordertype=order_type,
+    order = exchange.create_order(pair='ETH/BTC', ordertype=order_type, side="sell",
-                          amount=1, rate=200, time_in_force=time_in_force)
+                                  amount=1, rate=200, time_in_force=time_in_force)
 
     assert 'id' in order
     assert 'info' in order
@@ -1299,8 +1308,8 @@ def test_sell_considers_time_in_force(default_conf, mocker, exchange_name):
 
     order_type = 'market'
     time_in_force = 'ioc'
-    order = exchange.sell(pair='ETH/BTC', ordertype=order_type,
+    order = exchange.create_order(pair='ETH/BTC', ordertype=order_type, side="sell",
-                          amount=1, rate=200, time_in_force=time_in_force)
+                                  amount=1, rate=200, time_in_force=time_in_force)
 
     assert 'id' in order
     assert 'info' in order
@@ -1555,13 +1564,16 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None:
     pairs = [('IOTA/ETH', '5m'), ('XRP/ETH', '5m')]
     # empty dicts
     assert not exchange._klines
-    exchange.refresh_latest_ohlcv(pairs, cache=False)
+    res = exchange.refresh_latest_ohlcv(pairs, cache=False)
     # No caching
     assert not exchange._klines
+
+    assert len(res) == len(pairs)
     assert exchange._api_async.fetch_ohlcv.call_count == 2
     exchange._api_async.fetch_ohlcv.reset_mock()
 
-    exchange.refresh_latest_ohlcv(pairs)
+    res = exchange.refresh_latest_ohlcv(pairs)
+    assert len(res) == len(pairs)
 
     assert log_has(f'Refreshing candle (OHLCV) data for {len(pairs)} pairs', caplog)
     assert exchange._klines
@@ -1578,12 +1590,16 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None:
         assert exchange.klines(pair, copy=False) is exchange.klines(pair, copy=False)
 
     # test caching
-    exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m')])
+    res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m')])
+    assert len(res) == len(pairs)
 
     assert exchange._api_async.fetch_ohlcv.call_count == 2
     assert log_has(f"Using cached candle (OHLCV) data for pair {pairs[0][0]}, "
                    f"timeframe {pairs[0][1]} ...",
                    caplog)
+    res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m'), ('XRP/ETH', '1d')],
+                                        cache=False)
+    assert len(res) == 3
 
 
 @pytest.mark.asyncio
@@ -1835,6 +1851,31 @@ def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask,
     assert log_has("Using cached sell rate for ETH/BTC.", caplog)
 
 
+@pytest.mark.parametrize("entry,side,ask,bid,last,last_ab,expected", [
+    ('buy', 'ask', None, 4, 4,  0, 4),  # ask not available
+    ('buy', 'ask', None, None, 4,  0, 4),  # ask not available
+    ('buy', 'bid', 6, None, 4,  0, 5),  # bid not available
+    ('buy', 'bid', None, None, 4,  0, 5),  # No rate available
+    ('sell', 'ask', None, 4, 4,  0, 4),  # ask not available
+    ('sell', 'ask', None, None, 4,  0, 4),  # ask not available
+    ('sell', 'bid', 6, None, 4,  0, 5),  # bid not available
+    ('sell', 'bid', None, None, 4,  0, 5),  # bid not available
+])
+def test_get_ticker_rate_error(mocker, entry, default_conf, caplog, side, ask, bid,
+                               last, last_ab, expected) -> None:
+    caplog.set_level(logging.DEBUG)
+    default_conf['bid_strategy']['ask_last_balance'] = last_ab
+    default_conf['bid_strategy']['price_side'] = side
+    default_conf['ask_strategy']['price_side'] = side
+    default_conf['ask_strategy']['ask_last_balance'] = last_ab
+    exchange = get_patched_exchange(mocker, default_conf)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_ticker',
+                 return_value={'ask': ask, 'last': last, 'bid': bid})
+
+    with pytest.raises(PricingError):
+        exchange.get_rate('ETH/BTC', refresh=True, side=entry)
+
+
 @pytest.mark.parametrize('side,expected', [
     ('bid', 0.043936),  # Value from order_book_l2 fiture - bids side
     ('ask', 0.043949),  # Value from order_book_l2 fiture - asks side
@@ -2173,7 +2214,7 @@ def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange
     pair = 'ETH/BTC'
 
     with pytest.raises(OperationalException,
-                       match="This exchange does not suport downloading Trades."):
+                       match="This exchange does not support downloading Trades."):
         exchange.get_historic_trades(pair, since=trades_history[0][0],
                                      until=trades_history[-1][0])
 
@@ -2186,7 +2227,7 @@ def test_cancel_order_dry_run(default_conf, mocker, exchange_name):
     assert exchange.cancel_order(order_id='123', pair='TKN/BTC') == {}
     assert exchange.cancel_stoploss_order(order_id='123', pair='TKN/BTC') == {}
 
-    order = exchange.buy('ETH/BTC', 'limit', 5, 0.55, 'gtc')
+    order = exchange.create_order('ETH/BTC', 'limit', "buy", 5, 0.55, 'gtc')
 
     cancel_order = exchange.cancel_order(order_id=order['id'], pair='ETH/BTC')
     assert order['id'] == cancel_order['id']

tests/exchange/test_kraken.py

@@ -31,8 +31,8 @@ def test_buy_kraken_trading_agreement(default_conf, mocker):
     mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken")
 
-    order = exchange.buy(pair='ETH/BTC', ordertype=order_type,
+    order = exchange.create_order(pair='ETH/BTC', ordertype=order_type, side="buy",
-                         amount=1, rate=200, time_in_force=time_in_force)
+                                  amount=1, rate=200, time_in_force=time_in_force)
 
     assert 'id' in order
     assert 'info' in order
@@ -63,7 +63,8 @@ def test_sell_kraken_trading_agreement(default_conf, mocker):
     mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken")
 
-    order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200)
+    order = exchange.create_order(pair='ETH/BTC', ordertype=order_type,
+                                  side="sell", amount=1, rate=200)
 
     assert 'id' in order
     assert 'info' in order

tests/optimize/__init__.py

@@ -18,6 +18,7 @@ class BTrade(NamedTuple):
     sell_reason: SellType
     open_tick: int
     close_tick: int
+    buy_tag: Optional[str] = None
 
 
 class BTContainer(NamedTuple):
@@ -44,10 +45,13 @@ def _get_frame_time_from_offset(offset):
 
 def _build_backtest_dataframe(data):
     columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'buy', 'sell']
+    columns = columns + ['buy_tag'] if len(data[0]) == 9 else columns
 
     frame = DataFrame.from_records(data, columns=columns)
     frame['date'] = frame['date'].apply(_get_frame_time_from_offset)
     # Ensure floats are in place
     for column in ['open', 'high', 'low', 'close', 'volume']:
         frame[column] = frame[column].astype('float64')
+    if 'buy_tag' not in columns:
+        frame['buy_tag'] = None
     return frame

tests/optimize/conftest.py

@@ -16,7 +16,7 @@ def hyperopt_conf(default_conf):
     hyperconf.update({
         'datadir': Path(default_conf['datadir']),
         'runmode': RunMode.HYPEROPT,
-        'hyperopt': 'DefaultHyperOpt',
+        'hyperopt': 'HyperoptTestSepFile',
         'hyperopt_loss': 'ShortTradeDurHyperOptLoss',
                          'hyperopt_path': str(Path(__file__).parent / 'hyperopts'),
                          'epochs': 1,

tests/optimize/hyperopts/hyperopt_test_sep_file.py

@@ -11,7 +11,7 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib
 from freqtrade.optimize.hyperopt_interface import IHyperOpt
 
 
-class DefaultHyperOpt(IHyperOpt):
+class HyperoptTestSepFile(IHyperOpt):
     """
     Default hyperopt provided by the Freqtrade bot.
     You can override it with your own Hyperopt