Merge pull request #7806 from freqtrade/new_release

New release 2022.11

.devcontainer/devcontainer.json

@@ -11,12 +11,14 @@
     "mounts": [
         "source=freqtrade-bashhistory,target=/home/ftuser/commandhistory,type=volume"
     ],
+    "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/freqtrade,type=bind,consistency=cached",
     // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root.
     "remoteUser": "ftuser",
 
+    "onCreateCommand": "pip install --user -e .",
     "postCreateCommand": "freqtrade create-userdir --userdir user_data/",
 
-    "workspaceFolder": "/freqtrade/",
+    "workspaceFolder": "/workspaces/freqtrade",
 
     "settings": {
         "terminal.integrated.shell.linux": "/bin/bash",

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-18.04, ubuntu-20.04, ubuntu-22.04 ]
-        python-version: ["3.8", "3.9", "3.10.6"]
+        python-version: ["3.8", "3.9", "3.10"]
 
     steps:
     - uses: actions/checkout@v3
@@ -74,7 +74,7 @@ jobs:
       if: matrix.python-version == '3.9' && matrix.os == 'ubuntu-22.04'
 
     - name: Coveralls
-      if: (runner.os == 'Linux' && matrix.python-version == '3.9')
+      if: (runner.os == 'Linux' && matrix.python-version == '3.10' && matrix.os == 'ubuntu-22.04')
       env:
         # Coveralls token. Not used as secret due to github not providing secrets to forked repositories
         COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu
@@ -121,7 +121,7 @@ jobs:
     strategy:
       matrix:
         os: [ macos-latest ]
-        python-version: ["3.8", "3.9", "3.10.6"]
+        python-version: ["3.8", "3.9", "3.10"]
 
     steps:
     - uses: actions/checkout@v3
@@ -205,7 +205,7 @@ jobs:
     strategy:
       matrix:
         os: [ windows-latest ]
-        python-version: ["3.8", "3.9", "3.10.6"]
+        python-version: ["3.8", "3.9", "3.10"]
 
     steps:
     - uses: actions/checkout@v3
@@ -258,7 +258,7 @@ jobs:
           webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
 
   mypy_version_check:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
 
@@ -283,7 +283,7 @@ jobs:
     - uses: pre-commit/action@v3.0.0
 
   docs_check:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
 
@@ -313,7 +313,7 @@ jobs:
   # Notify only once - when CI completes (and after deploy) in case it's successfull
   notify-complete:
     needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check, pre-commit ]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     # Discord notification can't handle schedule events
     if: (github.event_name != 'schedule')
     permissions:
@@ -338,7 +338,7 @@ jobs:
 
   deploy:
     needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check, pre-commit ]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade'
 
@@ -441,4 +441,4 @@ jobs:
       with:
           severity: info
           details: Deploy Succeeded!
-          webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
+          webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}

.gitignore

@@ -109,7 +109,6 @@ target/
 !*.gitkeep
 !config_examples/config_binance.example.json
 !config_examples/config_bittrex.example.json
-!config_examples/config_ftx.example.json
 !config_examples/config_full.example.json
 !config_examples/config_kraken.example.json
 !config_examples/config_freqai.example.json

.pre-commit-config.yaml

@@ -15,9 +15,9 @@ repos:
         additional_dependencies:
           - types-cachetools==5.2.1
           - types-filelock==3.2.7
-          - types-requests==2.28.11
-          - types-tabulate==0.8.11
-          - types-python-dateutil==2.8.19
+          - types-requests==2.28.11.5
+          - types-tabulate==0.9.0.0
+          - types-python-dateutil==2.8.19.4
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort

README.md

@@ -28,7 +28,6 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even
 
 - [X] [Binance](https://www.binance.com/)
 - [X] [Bittrex](https://bittrex.com/)
-- [X] [FTX](https://ftx.com/#a=2258149)
 - [X] [Gate.io](https://www.gate.io/ref/6266643)
 - [X] [Huobi](http://huobi.com/)
 - [X] [Kraken](https://kraken.com/)
@@ -39,7 +38,7 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even
 
 - [X] [Binance](https://www.binance.com/)
 - [X] [Gate.io](https://www.gate.io/ref/6266643)
-- [X] [OKX](https://okx.com/).
+- [X] [OKX](https://okx.com/)
 
 Please make sure to read the [exchange specific notes](docs/exchanges.md), as well as the [trading with leverage](docs/leverage.md) documentation before diving in.
 

config_examples/config_binance.example.json

@@ -53,7 +53,7 @@
             "XTZ/BTC"
         ],
         "pair_blacklist": [
-            "BNB/BTC"
+            "BNB/.*"
         ]
     },
     "pairlists": [

config_examples/config_freqai.example.json

@@ -18,13 +18,8 @@
         "name": "binance",
         "key": "",
         "secret": "",
-        "ccxt_config": {
-            "enableRateLimit": true
-        },
-        "ccxt_async_config": {
-            "enableRateLimit": true,
-            "rateLimit": 200
-        },
+        "ccxt_config": {},
+        "ccxt_async_config": {},
         "pair_whitelist": [
             "1INCH/USDT",
             "ALGO/USDT"

config_examples/config_ftx.example.json

@@ -1,96 +0,0 @@
-{
-    "max_open_trades": 3,
-    "stake_currency": "USD",
-    "stake_amount": 50,
-    "tradable_balance_ratio": 0.99,
-    "fiat_display_currency": "USD",
-    "timeframe": "5m",
-    "dry_run": true,
-    "cancel_open_orders_on_exit": false,
-    "unfilledtimeout": {
-        "entry": 10,
-        "exit": 10,
-        "exit_timeout_count": 0,
-        "unit": "minutes"
-    },
-    "entry_pricing": {
-        "price_side": "same",
-        "use_order_book": true,
-        "order_book_top": 1,
-        "price_last_balance": 0.0,
-        "check_depth_of_market": {
-            "enabled": false,
-            "bids_to_ask_delta": 1
-        }
-    },
-    "exit_pricing": {
-        "price_side": "same",
-        "use_order_book": true,
-        "order_book_top": 1
-    },
-    "exchange": {
-        "name": "ftx",
-        "key": "your_exchange_key",
-        "secret": "your_exchange_secret",
-        "ccxt_config": {},
-        "ccxt_async_config": {},
-        "pair_whitelist": [
-            "BTC/USD",
-            "ETH/USD",
-            "BNB/USD",
-            "USDT/USD",
-            "LTC/USD",
-            "SRM/USD",
-            "SXP/USD",
-            "XRP/USD",
-            "DOGE/USD",
-            "1INCH/USD",
-            "CHZ/USD",
-            "MATIC/USD",
-            "LINK/USD",
-            "OXY/USD",
-            "SUSHI/USD"
-        ],
-        "pair_blacklist": [
-            "FTT/USD"
-        ]
-    },
-    "pairlists": [
-        {"method": "StaticPairList"}
-    ],
-    "edge": {
-        "enabled": false,
-        "process_throttle_secs": 3600,
-        "calculate_since_number_of_days": 7,
-        "allowed_risk": 0.01,
-        "stoploss_range_min": -0.01,
-        "stoploss_range_max": -0.1,
-        "stoploss_range_step": -0.01,
-        "minimum_winrate": 0.60,
-        "minimum_expectancy": 0.20,
-        "min_trade_number": 10,
-        "max_trade_duration_minute": 1440,
-        "remove_pumps": false
-    },
-    "telegram": {
-        "enabled": false,
-        "token": "your_telegram_token",
-        "chat_id": "your_telegram_chat_id"
-    },
-    "api_server": {
-        "enabled": false,
-        "listen_ip_address": "127.0.0.1",
-        "listen_port": 8080,
-        "verbosity": "error",
-        "jwt_secret_key": "somethingrandom",
-        "CORS_origins": [],
-        "username": "freqtrader",
-        "password": "SuperSecurePassword"
-    },
-    "bot_name": "freqtrade",
-    "initial_state": "running",
-    "force_entry_enable": false,
-    "internals": {
-        "process_throttle_secs": 5
-    }
-}

config_examples/config_full.example.json

@@ -204,6 +204,7 @@
     "strategy_path": "user_data/strategies/",
     "recursive_strategy_search": false,
     "add_config_files": [],
+    "reduce_df_footprint": false,
     "dataformat_ohlcv": "json",
     "dataformat_trades": "jsongz"
 }

docker/Dockerfile.armhf

@@ -11,7 +11,7 @@ ENV FT_APP_ENV="docker"
 # Prepare environment
 RUN mkdir /freqtrade \
   && apt-get update \
-  && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-dev \
+  && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-dev libutf8proc-dev libsnappy-dev \
   && apt-get clean \
   && useradd -u 1000 -G sudo -U -m ftuser \
   && chown ftuser:ftuser /freqtrade \
@@ -37,6 +37,7 @@ ENV LD_LIBRARY_PATH /usr/local/lib
 COPY --chown=ftuser:ftuser requirements.txt /freqtrade/
 USER ftuser
 RUN  pip install --user --no-cache-dir numpy \
+  && pip install --user /tmp/pyarrow-*.whl \
   && pip install --user --no-cache-dir -r requirements.txt
 
 # Copy dependencies to runtime-image

docs/advanced-hyperopt.md

@@ -78,6 +78,8 @@ This function needs to return a floating point number (`float`). Smaller numbers
 To override a pre-defined space (`roi_space`, `generate_roi_table`, `stoploss_space`, `trailing_space`), define a nested class called Hyperopt and define the required spaces as follows:
 
 ```python
+from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal
+
 class MyAwesomeStrategy(IStrategy):
     class HyperOpt:
         # Define a custom stoploss space.
@@ -94,6 +96,33 @@ class MyAwesomeStrategy(IStrategy):
                 SKDecimal(0.01, 0.07, decimals=3, name='roi_p2'),
                 SKDecimal(0.01, 0.20, decimals=3, name='roi_p3'),
             ]
+
+        def generate_roi_table(params: Dict) -> Dict[int, float]:
+
+            roi_table = {}
+            roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3']
+            roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2']
+            roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1']
+            roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0
+
+            return roi_table
+
+        def trailing_space() -> List[Dimension]:
+            # All parameters here are mandatory, you can only modify their type or the range.
+            return [
+                # Fixed to true, if optimizing trailing_stop we assume to use trailing stop at all times.
+                Categorical([True], name='trailing_stop'),
+
+                SKDecimal(0.01, 0.35, decimals=3, name='trailing_stop_positive'),
+                # 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive',
+                # so this intermediate parameter is used as the value of the difference between
+                # them. The value of the 'trailing_stop_positive_offset' is constructed in the
+                # generate_trailing_params() method.
+                # This is similar to the hyperspace dimensions used for constructing the ROI tables.
+                SKDecimal(0.001, 0.1, decimals=3, name='trailing_stop_positive_offset_p1'),
+
+                Categorical([True, False], name='trailing_only_offset_is_reached'),
+        ]
 ```
 
 !!! Note

docs/backtesting.md

@@ -522,13 +522,13 @@ Since backtesting lacks some detailed information about what happens within a ca
 - ROI
   - exits are compared to high - but the ROI value is used (e.g. ROI = 2%, high=5% - so the exit will be at 2%)
   - exits are never "below the candle", so a ROI of 2% may result in a exit at 2.4% if low was at 2.4% profit
-  - Forceexits caused by `<N>=-1` ROI entries use low as exit value, unless N falls on the candle open (e.g. `120: -1` for 1h candles)
+  - Force-exits caused by `<N>=-1` ROI entries use low as exit value, unless N falls on the candle open (e.g. `120: -1` for 1h candles)
 - Stoploss exits happen exactly at stoploss price, even if low was lower, but the loss will be `2 * fees` higher than the stoploss price
 - Stoploss is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` exit reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes
 - Low happens before high for stoploss, protecting capital first
 - Trailing stoploss
   - Trailing Stoploss is only adjusted if it's below the candle's low (otherwise it would be triggered)
-  - On trade entry candles that trigger trailing stoploss, the "minimum offset" (`stop_positive_offset`) is assumed (instead of high) - and the stop is calculated from this point
+  - On trade entry candles that trigger trailing stoploss, the "minimum offset" (`stop_positive_offset`) is assumed (instead of high) - and the stop is calculated from this point. This rule is NOT applicable to custom-stoploss scenarios, since there's no information about the stoploss logic available.
   - High happens first - adjusting stoploss
   - Low uses the adjusted stoploss (so exits with large high-low difference are backtested correctly)
   - ROI applies before trailing-stop, ensuring profits are "top-capped" at ROI if both ROI and trailing stop applies
@@ -546,8 +546,8 @@ In addition to the above assumptions, strategy authors should carefully read the
 
 ### Trading limits in backtesting
 
-Exchanges have certain trading limits, like minimum base currency, or minimum stake (quote) currency.
-These limits are usually listed in the exchange documentation as "trading rules" or similar.
+Exchanges have certain trading limits, like minimum (and maximum) base currency, or minimum/maximum stake (quote) currency.
+These limits are usually listed in the exchange documentation as "trading rules" or similar and can be quite different between different pairs.
 
 Backtesting (as well as live and dry-run) does honor these limits, and will ensure that a stoploss can be placed below this value - so the value will be slightly higher than what the exchange specifies.
 Freqtrade has however no information about historic limits.

docs/configuration.md

@@ -215,16 +215,18 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `telegram.balance_dust_level` | Dust-level (in stake currency) - currencies with a balance below this will not be shown by `/balance`. <br> **Datatype:** float
 | `telegram.reload` | Allow "reload" buttons on telegram messages. <br>*Defaults to `True`.<br> **Datatype:** boolean
 | `telegram.notification_settings.*` | Detailed notification settings. Refer to the [telegram documentation](telegram-usage.md) for details.<br> **Datatype:** dictionary
+| `telegram.allow_custom_messages` | Enable the sending of Telegram messages from strategies via the dataprovider.send_msg() function. <br> **Datatype:** Boolean
 | | **Webhook**
 | `webhook.enabled` | Enable usage of Webhook notifications <br> **Datatype:** Boolean
 | `webhook.url` | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
-| `webhook.webhookentry` | Payload to send on entry. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
-| `webhook.webhookentrycancel` | Payload to send on entry order cancel. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
-| `webhook.webhookentryfill` | Payload to send on entry order filled. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
-| `webhook.webhookexit` | Payload to send on exit. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
-| `webhook.webhookexitcancel` | Payload to send on exit order cancel. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
-| `webhook.webhookexitfill` | Payload to send on exit order filled. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
-| `webhook.webhookstatus` | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
+| `webhook.entry` | Payload to send on entry. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
+| `webhook.entry_cancel` | Payload to send on entry order cancel. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
+| `webhook.entry_fill` | Payload to send on entry order filled. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
+| `webhook.exit` | Payload to send on exit. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
+| `webhook.exit_cancel` | Payload to send on exit order cancel. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
+| `webhook.exit_fill` | Payload to send on exit order filled. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
+| `webhook.status` | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. <br> **Datatype:** String
+| `webhook.allow_custom_messages` | Enable the sending of Webhook messages from strategies via the dataprovider.send_msg() function. <br> **Datatype:** Boolean
 | | **Rest API / FreqUI / Producer-Consumer**
 | `api_server.enabled` | Enable usage of API Server. See the [API Server documentation](rest-api.md) for more details. <br> **Datatype:** Boolean
 | `api_server.listen_ip_address` | Bind IP address. See the [API Server documentation](rest-api.md) for more details. <br> **Datatype:** IPv4
@@ -251,6 +253,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `add_config_files` | Additional config files. These files will be loaded and merged with the current config file. The files are resolved relative to the initial file.<br> *Defaults to `[]`*. <br> **Datatype:** List of strings
 | `dataformat_ohlcv` | Data format to use to store historical candle (OHLCV) data. <br> *Defaults to `json`*. <br> **Datatype:** String
 | `dataformat_trades` | Data format to use to store historical trades data. <br> *Defaults to `jsongz`*. <br> **Datatype:** String
+| `reduce_df_footprint` | Recast all numeric columns to float32/int32, with the objective of reducing ram/disk usage (and decreasing train/inference timing in FreqAI). (Currently only affects FreqAI use-cases) <br> **Datatype:** Boolean. <br> Default: `False`.
 
 ### Parameters in the strategy
 
@@ -550,7 +553,7 @@ The possible values are: `GTC` (default), `FOK` or `IOC`.
 ```
 
 !!! Warning
-    This is ongoing work. For now, it is supported only for binance, gate, ftx and kucoin.
+    This is ongoing work. For now, it is supported only for binance, gate and kucoin.
     Please don't change the default value unless you know what you are doing and have researched the impact of using different values for your particular exchange.
 
 ### What values can be used for fiat_display_currency?
@@ -662,6 +665,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d
 ### Using proxy with Freqtrade
 
 To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values.
+This will have the proxy settings applied to everything (telegram, coingecko, ...) except exchange requests.
 
 ``` bash
 export HTTP_PROXY="http://addr:port"
@@ -669,17 +673,20 @@ export HTTPS_PROXY="http://addr:port"
 freqtrade
 ```
 
-#### Proxy just exchange requests
+#### Proxy exchange requests
 
-To use a proxy just for exchange connections (skips/ignores telegram and coingecko) - you can also define the proxies as part of the ccxt configuration.
+To use a proxy for exchange connections - you will have to define the proxies as part of the ccxt configuration.
 
 ``` json
-"ccxt_config": {
+{ 
+  "exchange": {
+    "ccxt_config": {
     "aiohttp_proxy": "http://addr:port",
     "proxies": {
-        "http": "http://addr:port",
-        "https": "http://addr:port"
+      "http": "http://addr:port",
+      "https": "http://addr:port"
     },
+  }
 }
 ```
 

docs/data-download.md

@@ -177,13 +177,13 @@ freqtrade download-data --exchange binance --pairs ETH/USDT XRP/USDT BTC/USDT --
 
 ### Data format
 
-Freqtrade currently supports 3 data-formats for both OHLCV and trades data:
+Freqtrade currently supports the following data-formats:
 
 * `json` -  plain "text" json files
 * `jsongz` - a gzip-zipped version of json files
 * `hdf5` - a high performance datastore
-* `feather` - a dataformat based on Apache Arrow
-* `parquet` - columnar datastore
+* `feather` - a dataformat based on Apache Arrow (OHLCV only)
+* `parquet` - columnar datastore (OHLCV only)
 
 By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data.
 

docs/deprecated.md

@@ -66,11 +66,11 @@ We will keep a compatibility layer for 1-2 versions (so both `buy_tag` and `ente
 
 #### Naming changes
 
-Webhook terminology changed from "sell" to "exit", and from "buy" to "entry".
+Webhook terminology changed from "sell" to "exit", and from "buy" to "entry", removing "webhook" in the process.
 
-* `webhookbuy` -> `webhookentry`
-* `webhookbuyfill` -> `webhookentryfill`
-* `webhookbuycancel` -> `webhookentrycancel`
-* `webhooksell` -> `webhookexit`
-* `webhooksellfill` -> `webhookexitfill`
-* `webhooksellcancel` -> `webhookexitcancel`
+* `webhookbuy`, `webhookentry` -> `entry`
+* `webhookbuyfill`, `webhookentryfill` -> `entry_fill`
+* `webhookbuycancel`, `webhookentrycancel` -> `entry_cancel`
+* `webhooksell`, `webhookexit` -> `exit`
+* `webhooksellfill`, `webhookexitfill` -> `exit_fill`
+* `webhooksellcancel`, `webhookexitcancel` -> `exit_cancel`

docs/developer.md

@@ -434,6 +434,11 @@ To keep the release-log short, best wrap the full git changelog into a collapsib
 </details>
 ```
 
+### FreqUI release
+
+If FreqUI has been updated substantially, make sure to create a release before merging the release branch.
+Make sure that freqUI CI on the release is finished and passed before merging the release.
+
 ### Create github release / tag
 
 Once the PR against stable is merged (best right after merging):

docs/exchanges.md

@@ -60,11 +60,18 @@ Binance supports [time_in_force](configuration.md#understand-order_time_in_force
     Binance supports `stoploss_on_exchange` and uses `stop-loss-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange.
     On futures, Binance supports both `stop-limit` as well as `stop-market` orders. You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type to use.
 
-### Binance Blacklist
+### Binance Blacklist recommendation
 
 For Binance, it is suggested to add `"BNB/<STAKE>"` to your blacklist to avoid issues, unless you are willing to maintain enough extra `BNB` on the account or unless you're willing to disable using `BNB` for fees.
 Binance accounts may use `BNB` for fees, and if a trade happens to be on `BNB`, further trades may consume this position and make the initial BNB trade unsellable as the expected amount is not there anymore.
 
+### Binance sites
+
+Binance has been split into 2, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized.
+
+* [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance`.
+* [binance.us](https://www.binance.us/) - US based users. Use exchange id: `binanceus`.
+
 ### Binance Futures
 
 Binance has specific (unfortunately complex) [Futures Trading Quantitative Rules](https://www.binance.com/en/support/faq/4f462ebe6ff445d4a170be7d9e897272) which need to be followed, and which prohibit a too low stake-amount (among others) for too many orders.
@@ -87,12 +94,14 @@ When trading on Binance Futures market, orderbook must be used because there is
   },
 ```
 
-### Binance sites
+#### Binance futures settings
 
-Binance has been split into 2, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized.
+Users will also have to have the futures-setting "Position Mode" set to "One-way Mode", and "Asset Mode" set to "Single-Asset Mode".
+These settings will be checked on startup, and freqtrade will show an error if this setting is wrong.
 
-* [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance`.
-* [binance.us](https://www.binance.us/) - US based users. Use exchange id: `binanceus`.
+![Binance futures settings](assets/binance_futures_settings.png)
+
+Freqtrade will not attempt to change these settings.
 
 ## Kraken
 
@@ -164,26 +173,6 @@ res = [p for p, x in lm.items() if 'US' in x['info']['prohibitedIn']]
 print(res)
 ```
 
-## FTX
-
-!!! Tip "Stoploss on Exchange"
-    FTX supports `stoploss_on_exchange` and can use both stop-loss-market and stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it.
-    You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type of stoploss shall be used.
-
-### Using subaccounts
-
-To use subaccounts with FTX, you need to edit the configuration and add the following:
-
-``` json
-"exchange": {
-    "ccxt_config": {
-        "headers": {
-            "FTX-SUBACCOUNT": "name"
-        }
-    },
-}
-```
-
 ## Kucoin
 
 Kucoin requires a passphrase for each api key, you will therefore need to add this key into the configuration so your exchange section looks as follows:

docs/faq.md

@@ -102,6 +102,12 @@ If this happens for all pairs in the pairlist, this might indicate a recent exch
 
 Irrespectively of the reason, Freqtrade will fill up these candles with "empty" candles, where open, high, low and close are set to the previous candle close - and volume is empty. In a chart, this will look like a `_` - and is aligned with how exchanges usually represent 0 volume candles.
 
+### I'm getting "Price jump between 2 candles detected"
+
+This message is a warning that the candles had a price jump of > 30%.
+This might be a sign that the pair stopped trading, and some token exchange took place (e.g. COCOS in 2021 - where price jumped from 0.0000154 to 0.01621).
+This message is often accompanied by ["Missing data fillup"](#im-getting-missing-data-fillup-messages-in-the-log) - as trading on such pairs is often stopped for some time.
+
 ### I'm getting "Outdated history for pair xxx" in the log
 
 The bot is trying to tell you that it got an outdated last candle (not the last complete candle).

docs/freqai-configuration.md

@@ -1,10 +1,10 @@
 # Configuration
 
-`FreqAI` is configured through the typical [Freqtrade config file](configuration.md) and the standard [Freqtrade strategy](strategy-customization.md). Examples of `FreqAI` config and strategy files can be found in `config_examples/config_freqai.example.json` and `freqtrade/templates/FreqaiExampleStrategy.py`, respectively.
+FreqAI is configured through the typical [Freqtrade config file](configuration.md) and the standard [Freqtrade strategy](strategy-customization.md). Examples of FreqAI config and strategy files can be found in `config_examples/config_freqai.example.json` and `freqtrade/templates/FreqaiExampleStrategy.py`, respectively.
 
 ## Setting up the configuration file
 
- Although there are plenty of additional parameters to choose from, as highlighted in the [parameter table](freqai-parameter-table.md#parameter-table), a `FreqAI` config must at minimum include the following parameters (the parameter values are only examples):
+ Although there are plenty of additional parameters to choose from, as highlighted in the [parameter table](freqai-parameter-table.md#parameter-table), a FreqAI config must at minimum include the following parameters (the parameter values are only examples):
 
 ```json
     "freqai": {
@@ -35,9 +35,9 @@
 
 A full example config is available in `config_examples/config_freqai.example.json`.
 
-## Building a `FreqAI` strategy
+## Building a FreqAI strategy
 
-The `FreqAI` strategy requires including the following lines of code in the standard [Freqtrade strategy](strategy-customization.md):
+The FreqAI strategy requires including the following lines of code in the standard [Freqtrade strategy](strategy-customization.md):
 
 ```python
     # user should define the maximum startup candle count (the largest number of candles
@@ -61,7 +61,7 @@ The `FreqAI` strategy requires including the following lines of code in the stan
         """
         Function designed to automatically generate, name and merge features
         from user indicated timeframes in the configuration file. User controls the indicators
-        passed to the training/prediction by prepending indicators with `'%-' + coin `
+        passed to the training/prediction by prepending indicators with `'%-' + pair `
         (see convention below). I.e. user should not prepend any supporting metrics
         (e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the
         model.
@@ -69,20 +69,17 @@ The `FreqAI` strategy requires including the following lines of code in the stan
         :param df: strategy dataframe which will receive merges from informatives
         :param tf: timeframe of the dataframe which will modify the feature names
         :param informative: the dataframe associated with the informative pair
-        :param coin: the name of the coin which will modify the feature names.
         """
 
-        coin = pair.split('/')[0]
-
         if informative is None:
             informative = self.dp.get_pair_dataframe(pair, tf)
 
         # first loop is automatically duplicating indicators for time periods
         for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
             t = int(t)
-            informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
-            informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
-            informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t)
+            informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
+            informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
+            informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, window=t)
 
         indicators = [col for col in informative if col.startswith("%")]
         # This loop duplicates and shifts all indicators to add a sense of recency to data
@@ -129,12 +126,12 @@ Notice also the location of the labels under `if set_generalized_indicators:` at
     The `self.freqai.start()` function cannot be called outside the `populate_indicators()`.
 
 !!! Note
-    Features **must** be defined in `populate_any_indicators()`. Defining `FreqAI` features in `populate_indicators()`
+    Features **must** be defined in `populate_any_indicators()`. Defining FreqAI features in `populate_indicators()`
     will cause the algorithm to fail in live/dry mode. In order to add generalized features that are not associated with a specific pair or timeframe, the following structure inside `populate_any_indicators()` should be used
     (as exemplified in `freqtrade/templates/FreqaiExampleStrategy.py`):
 
     ```python
-        def populate_any_indicators(self, metadata, pair, df, tf, informative=None, coin="", set_generalized_indicators=False):
+        def populate_any_indicators(self, pair, df, tf, informative=None, set_generalized_indicators=False):
 
             ...
 
@@ -166,15 +163,15 @@ Below are the values you can expect to include/use inside a typical strategy dat
 
 |  DataFrame Key | Description |
 |------------|-------------|
-| `df['&*']` | Any dataframe column prepended with `&` in `populate_any_indicators()` is treated as a training target (label) inside `FreqAI` (typically following the naming convention `&-s*`). The names of these dataframe columns are fed back as the predictions. For example, to predict the price change in the next 40 candles (similar to `templates/FreqaiExampleStrategy.py`), you would set `df['&-s_close']`. `FreqAI` makes the predictions and gives them back under the same key (`df['&-s_close']`) to be used in `populate_entry/exit_trend()`. <br> **Datatype:** Depends on the output of the model.
+| `df['&*']` | Any dataframe column prepended with `&` in `populate_any_indicators()` is treated as a training target (label) inside FreqAI (typically following the naming convention `&-s*`). For example, to predict the close price 40 candles into the future, you would set `df['&-s_close'] = df['close'].shift(-self.freqai_info["feature_parameters"]["label_period_candles"])` with `"label_period_candles": 40` in the config. FreqAI makes the predictions and gives them back under the same key (`df['&-s_close']`) to be used in `populate_entry/exit_trend()`. <br> **Datatype:** Depends on the output of the model.
 | `df['&*_std/mean']` | Standard deviation and mean values of the defined labels during training (or live tracking with `fit_live_predictions_candles`). Commonly used to understand the rarity of a prediction (use the z-score as shown in `templates/FreqaiExampleStrategy.py` and explained [here](#creating-a-dynamic-target-threshold) to evaluate how often a particular prediction was observed during training or historically with `fit_live_predictions_candles`). <br> **Datatype:** Float.
-| `df['do_predict']` | Indication of an outlier data point. The return value is integer between -1 and 2, which lets you know if the prediction is trustworthy or not. `do_predict==1` means that the prediction is trustworthy. If the Dissimilarity Index (DI, see details [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di)) of the input data point is above the threshold defined in the config, `FreqAI` will subtract 1 from `do_predict`, resulting in `do_predict==0`. If `use_SVM_to_remove_outliers()` is active, the Support Vector Machine (SVM, see details [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm)) may also detect outliers in training and prediction data. In this case, the SVM will also subtract 1 from `do_predict`. If the input data point was considered an outlier by the SVM but not by the DI, or vice versa, the result will be `do_predict==0`. If both the DI and the SVM considers the input data point to be an outlier, the result will be `do_predict==-1`. A particular case is when `do_predict == 2`, which means that the model has expired due to exceeding `expired_hours`. <br> **Datatype:** Integer between -1 and 2.
-| `df['DI_values']` | Dissimilarity Index (DI) values are proxies for the level of confidence `FreqAI` has in the prediction. A lower DI means the prediction is close to the training data, i.e., higher prediction confidence. See details about the DI [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di). <br> **Datatype:** Float.
-| `df['%*']` | Any dataframe column prepended with `%` in `populate_any_indicators()` is treated as a training feature. For example, you can include the RSI in the training feature set (similar to in `templates/FreqaiExampleStrategy.py`) by setting `df['%-rsi']`. See more details on how this is done [here](freqai-feature-engineering.md). <br> **Note:** Since the number of features prepended with `%` can multiply very quickly (10s of thousands of features is easily engineered using the multiplictative functionality described in the `feature_parameters` table shown above), these features are removed from the dataframe upon return from `FreqAI`. To keep a particular type of feature for plotting purposes, you would prepend it with `%%`. <br> **Datatype:** Depends on the output of the model.
+| `df['do_predict']` | Indication of an outlier data point. The return value is integer between -2 and 2, which lets you know if the prediction is trustworthy or not. `do_predict==1` means that the prediction is trustworthy. If the Dissimilarity Index (DI, see details [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di)) of the input data point is above the threshold defined in the config, FreqAI will subtract 1 from `do_predict`, resulting in `do_predict==0`. If `use_SVM_to_remove_outliers()` is active, the Support Vector Machine (SVM, see details [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm)) may also detect outliers in training and prediction data. In this case, the SVM will also subtract 1 from `do_predict`. If the input data point was considered an outlier by the SVM but not by the DI, or vice versa, the result will be `do_predict==0`. If both the DI and the SVM considers the input data point to be an outlier, the result will be `do_predict==-1`. As with the SVM, if `use_DBSCAN_to_remove_outliers` is active, DBSCAN (see details [here](freqai-feature-engineering.md#identifying-outliers-with-dbscan)) may also detect outliers and subtract 1 from `do_predict`. Hence, if both the SVM and DBSCAN are active and identify a datapoint that was above the DI threshold as an outlier, the result will be `do_predict==-2`. A particular case is when `do_predict == 2`, which means that the model has expired due to exceeding `expired_hours`. <br> **Datatype:** Integer between -2 and 2.
+| `df['DI_values']` | Dissimilarity Index (DI) values are proxies for the level of confidence FreqAI has in the prediction. A lower DI means the prediction is close to the training data, i.e., higher prediction confidence. See details about the DI [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di). <br> **Datatype:** Float.
+| `df['%*']` | Any dataframe column prepended with `%` in `populate_any_indicators()` is treated as a training feature. For example, you can include the RSI in the training feature set (similar to in `templates/FreqaiExampleStrategy.py`) by setting `df['%-rsi']`. See more details on how this is done [here](freqai-feature-engineering.md). <br> **Note:** Since the number of features prepended with `%` can multiply very quickly (10s of thousands of features are easily engineered using the multiplictative functionality of, e.g., `include_shifted_candles` and `include_timeframes` as described in the [parameter table](freqai-parameter-table.md)), these features are removed from the dataframe that is returned from FreqAI to the strategy. To keep a particular type of feature for plotting purposes, you would prepend it with `%%`. <br> **Datatype:** Depends on the output of the model.
 
 ## Setting the `startup_candle_count`
 
-The `startup_candle_count` in the `FreqAI` strategy needs to be set up in the same way as in the standard Freqtrade strategy (see details [here](strategy-customization.md#strategy-startup-period)). This value is used by Freqtrade to ensure that a sufficient amount of data is provided when calling the `dataprovider`, to avoid any NaNs at the beginning of the first training. You can easily set this value by identifying the longest period (in candle units) which is passed to the indicator creation functions (e.g., Ta-Lib functions). In the presented example, `startup_candle_count` is 20 since this is the maximum value in `indicators_periods_candles`.
+The `startup_candle_count` in the FreqAI strategy needs to be set up in the same way as in the standard Freqtrade strategy (see details [here](strategy-customization.md#strategy-startup-period)). This value is used by Freqtrade to ensure that a sufficient amount of data is provided when calling the `dataprovider`, to avoid any NaNs at the beginning of the first training. You can easily set this value by identifying the longest period (in candle units) which is passed to the indicator creation functions (e.g., Ta-Lib functions). In the presented example, `startup_candle_count` is 20 since this is the maximum value in `indicators_periods_candles`.
 
 !!! Note
     There are instances where the Ta-Lib functions actually require more data than just the passed `period` or else the feature dataset gets populated with NaNs. Anecdotally, multiplying the `startup_candle_count` by 2 always leads to a fully NaN free training dataset. Hence, it is typically safest to multiply the expected `startup_candle_count` by 2. Look out for this log message to confirm that the data is clean:
@@ -185,33 +182,63 @@ The `startup_candle_count` in the `FreqAI` strategy needs to be set up in the sa
 
 ## Creating a dynamic target threshold
 
-Deciding when to enter or exit a trade can be done in a dynamic way to reflect current market conditions. `FreqAI` allows you to return additional information from the training of a model (more info [here](freqai-feature-engineering.md#returning-additional-info-from-training)). For example, the `&*_std/mean` return values describe the statistical distribution of the target/label *during the most recent training*. Comparing a given prediction to these values allows you to know the rarity of the prediction. In `templates/FreqaiExampleStrategy.py`, the `target_roi` and  `sell_roi` are defined to be 1.25 z-scores away from the mean which causes predictions that are closer to the mean to be filtered out. 
+Deciding when to enter or exit a trade can be done in a dynamic way to reflect current market conditions. FreqAI allows you to return additional information from the training of a model (more info [here](freqai-feature-engineering.md#returning-additional-info-from-training)). For example, the `&*_std/mean` return values describe the statistical distribution of the target/label *during the most recent training*. Comparing a given prediction to these values allows you to know the rarity of the prediction. In `templates/FreqaiExampleStrategy.py`, the `target_roi` and  `sell_roi` are defined to be 1.25 z-scores away from the mean which causes predictions that are closer to the mean to be filtered out. 
 
 ```python
 dataframe["target_roi"] = dataframe["&-s_close_mean"] + dataframe["&-s_close_std"] * 1.25
 dataframe["sell_roi"] = dataframe["&-s_close_mean"] - dataframe["&-s_close_std"] * 1.25
 ```
 
-To consider the population of *historical predictions* for creating the dynamic target instead of information from the training as discussed above, you would set `fit_live_prediction_candles` in the config to the number of historical prediction candles you wish to use to generate target statistics.
+To consider the population of *historical predictions* for creating the dynamic target instead of information from the training as discussed above, you would set `fit_live_predictions_candles` in the config to the number of historical prediction candles you wish to use to generate target statistics.
 
 ```json
     "freqai": {
-        "fit_live_prediction_candles": 300,
+        "fit_live_predictions_candles": 300,
     }
 ```
 
-If this value is set, `FreqAI` will initially use the predictions from the training data and subsequently begin introducing real prediction data as it is generated. `FreqAI` will save this historical data to be reloaded if you stop and restart a model with the same `identifier`.
+If this value is set, FreqAI will initially use the predictions from the training data and subsequently begin introducing real prediction data as it is generated. FreqAI will save this historical data to be reloaded if you stop and restart a model with the same `identifier`.
 
 ## Using different prediction models
 
-`FreqAI` has multiple example prediction model libraries that are ready to be used as is via the flag `--freqaimodel`. These libraries include `Catboost`, `LightGBM`, and `XGBoost` regression, classification, and multi-target models, and can be found in `freqai/prediction_models/`. However, it is possible to customize and create your own prediction models using the `IFreqaiModel` class. You are encouraged to inherit `fit()`, `train()`, and `predict()` to let these customize various aspects of the training procedures.
+FreqAI has multiple example prediction model libraries that are ready to be used as is via the flag `--freqaimodel`. These libraries include `CatBoost`, `LightGBM`, and `XGBoost` regression, classification, and multi-target models, and can be found in `freqai/prediction_models/`.
 
-### Setting classifier targets
+Regression and classification models differ in what targets they predict - a regression model will predict a target of continuous values, for example what price BTC will be at tomorrow, whilst a classifier will predict a target of discrete values, for example if the price of BTC will go up tomorrow or not. This means that you have to specify your targets differently depending on which model type you are using (see details [below](#setting-model-targets)).
 
-`FreqAI` includes a variety of classifiers, such as the `CatboostClassifier` via the flag `--freqaimodel CatboostClassifier`. If you elects to use a classifier, the classes need to be set using strings. For example:
+All of the aforementioned model libraries implement gradient boosted decision tree algorithms. They all work on the principle of ensemble learning, where predictions from multiple simple learners are combined to get a final prediction that is more stable and generalized. The simple learners in this case are decision trees. Gradient boosting refers to the method of learning, where each simple learner is built in sequence - the subsequent learner is used to improve on the error from the previous learner. If you want to learn more about the different model libraries you can find the information in their respective docs:
+
+* CatBoost: https://catboost.ai/en/docs/
+* LightGBM: https://lightgbm.readthedocs.io/en/v3.3.2/#
+* XGBoost: https://xgboost.readthedocs.io/en/stable/#
+
+There are also numerous online articles describing and comparing the algorithms. Some relatively light-weight examples would be [CatBoost vs. LightGBM vs. XGBoost — Which is the best algorithm?](https://towardsdatascience.com/catboost-vs-lightgbm-vs-xgboost-c80f40662924#:~:text=In%20CatBoost%2C%20symmetric%20trees%2C%20or,the%20same%20depth%20can%20differ.) and [XGBoost, LightGBM or CatBoost — which boosting algorithm should I use?](https://medium.com/riskified-technology/xgboost-lightgbm-or-catboost-which-boosting-algorithm-should-i-use-e7fda7bb36bc). Keep in mind that the performance of each model is highly dependent on the application and so any reported metrics might not be true for your particular use of the model.
+
+Apart from the models already available in FreqAI, it is also possible to customize and create your own prediction models using the `IFreqaiModel` class. You are encouraged to inherit `fit()`, `train()`, and `predict()` to customize various aspects of the training procedures. You can place custom FreqAI models in `user_data/freqaimodels` - and freqtrade will pick them up from there based on the provided `--freqaimodel` name - which has to correspond to the class name of your custom model.
+Make sure to use unique names to avoid overriding built-in models.
+
+### Setting model targets
+
+#### Regressors
+
+If you are using a regressor, you need to specify a target that has continuous values. FreqAI includes a variety of regressors, such as the `CatboostRegressor`via the flag `--freqaimodel CatboostRegressor`. An example of how you could set a regression target for predicting the price 100 candles into the future would be
+
+```python
+df['&s-close_price'] = df['close'].shift(-100)
+```
+
+If you want to predict multiple targets, you need to define multiple labels using the same syntax as shown above.
+
+#### Classifiers
+
+If you are using a classifier, you need to specify a target that has discrete values. FreqAI includes a variety of classifiers, such as the `CatboostClassifier` via the flag `--freqaimodel CatboostClassifier`. If you elects to use a classifier, the classes need to be set using strings. For example, if you want to predict if the price 100 candles into the future goes up or down you would set 
 
 ```python
 df['&s-up_or_down'] = np.where( df["close"].shift(-100) > df["close"], 'up', 'down')
 ```
 
-Additionally, the example classifier models do not accommodate multiple labels, but they do allow multi-class classification within a single label column.
+If you want to predict multiple targets you must specify all labels in the same label column. You could, for example, add the label `same` to define where the price was unchanged by setting
+
+```python
+df['&s-up_or_down'] = np.where( df["close"].shift(-100) > df["close"], 'up', 'down')
+df['&s-up_or_down'] = np.where( df["close"].shift(-100) == df["close"], 'same', df['&s-up_or_down'])
+```

docs/freqai-developers.md

@@ -2,13 +2,13 @@
 
 ## Project architecture
 
-The architecture and functions of `FreqAI` are generalized to encourages development of unique features, functions, models, etc.
+The architecture and functions of FreqAI are generalized to encourages development of unique features, functions, models, etc.
 
 The class structure and a detailed algorithmic overview is depicted in the following diagram:
 
 ![image](assets/freqai_algorithm-diagram.jpg)
 
-As shown, there are three distinct objects comprising `FreqAI`:
+As shown, there are three distinct objects comprising FreqAI:
 
 * **IFreqaiModel** - A singular persistent object containing all the necessary logic to collect, store, and process data, engineer features, run training, and inference models.
 * **FreqaiDataKitchen** - A non-persistent object which is created uniquely for each unique asset/model. Beyond metadata, it also contains a variety of data processing tools.
@@ -18,7 +18,7 @@ There are a variety of built-in [prediction models](freqai-configuration.md#usin
 
 ## Data handling
 
-`FreqAI` aims to organize model files, prediction data, and meta data in a way that simplifies post-processing and enhances crash resilience by automatic data reloading. The data is saved in a file structure,`user_data_dir/models/`, which contains all the data associated with the trainings and backtests. The `FreqaiDataKitchen()` relies heavily on the file structure for proper training and inferencing and should therefore not be manually modified.
+FreqAI aims to organize model files, prediction data, and meta data in a way that simplifies post-processing and enhances crash resilience by automatic data reloading. The data is saved in a file structure,`user_data_dir/models/`, which contains all the data associated with the trainings and backtests. The `FreqaiDataKitchen()` relies heavily on the file structure for proper training and inferencing and should therefore not be manually modified.
 
 ### File structure
 
@@ -27,13 +27,13 @@ The file structure is automatically generated based on the model `identifier` se
 | Structure | Description |
 |-----------|-------------|
 | `config_*.json` | A copy of the model specific configuration file. |
-| `historic_predictions.pkl` | A file containing all historic predictions generated during the lifetime of the `identifier` model during live deployment. `historic_predictions.pkl` is used to reload the model after a crash or a config change. A backup file is always held incase of corruption on the main file. **`FreqAI` automatically detects corruption and replaces the corrupted file with the backup**. |
+| `historic_predictions.pkl` | A file containing all historic predictions generated during the lifetime of the `identifier` model during live deployment. `historic_predictions.pkl` is used to reload the model after a crash or a config change. A backup file is always held in case of corruption on the main file. FreqAI **automatically** detects corruption and replaces the corrupted file with the backup. |
 | `pair_dictionary.json` | A file containing the training queue as well as the on disk location of the most recently trained model. |
 | `sub-train-*_TIMESTAMP` | A folder containing all the files associated with a single model, such as: <br>
-|| `*_metadata.json` - Metadata for the model, such as normalization max/mins, expected training feature list, etc. <br>
+|| `*_metadata.json` - Metadata for the model, such as normalization max/min, expected training feature list, etc. <br>
 || `*_model.*` - The model file saved to disk for reloading from a crash. Can be `joblib` (typical boosting libs), `zip` (stable_baselines), `hd5` (keras type), etc. <br>
-|| `*_pca_object.pkl` - The [Principal component analysis (PCA)](freqai-feature-engineering.md#data-dimensionality-reduction-with-principal-component-analysis) transform (if `principal_component_analysis: true` is set in the config) which will be used to transform unseen prediction features. <br>
-|| `*_svm_model.pkl` - The [Support Vector Machine (SVM)](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm) model which is used to detect outliers in unseen prediction features. <br>
+|| `*_pca_object.pkl` - The [Principal component analysis (PCA)](freqai-feature-engineering.md#data-dimensionality-reduction-with-principal-component-analysis) transform (if `principal_component_analysis: True` is set in the config) which will be used to transform unseen prediction features. <br>
+|| `*_svm_model.pkl` - The [Support Vector Machine (SVM)](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm) model (if `use_SVM_to_remove_outliers: True` is set in the config) which is used to detect outliers in unseen prediction features. <br>
 || `*_trained_df.pkl` - The dataframe containing all the training features used to train the `identifier` model. This is used for computing the [Dissimilarity Index (DI)](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di) and can also be used for post-processing. <br>
 || `*_trained_dates.df.pkl` - The dates associated with the `trained_df.pkl`, which is useful for post-processing. |
 

docs/freqai-feature-engineering.md

@@ -2,9 +2,12 @@
 
 ## Defining the features
 
-Low level feature engineering is performed in the user strategy within a function called `populate_any_indicators()`. That function sets the `base features` such as, `RSI`, `MFI`, `EMA`, `SMA`, time of day, volume, etc. The `base features` can be custom indicators or they can be imported from any technical-analysis library that you can find. One important syntax rule is that all `base features` string names are prepended with `%`, while labels/targets are prepended with `&`.
+Low level feature engineering is performed in the user strategy within a function called `populate_any_indicators()`. That function sets the `base features` such as, `RSI`, `MFI`, `EMA`, `SMA`, time of day, volume, etc. The `base features` can be custom indicators or they can be imported from any technical-analysis library that you can find. One important syntax rule is that all `base features` string names are prepended with `%-{pair}`, while labels/targets are prepended with `&`.
 
-Meanwhile, high level feature engineering is handled within `"feature_parameters":{}` in the `FreqAI` config. Within this file, it is possible to decide large scale feature expansions on top of the `base_features` such as "including correlated pairs" or "including informative timeframes" or even "including recent candles." 
+!!! Note
+    Adding the full pair string, e.g. XYZ/USD, in the feature name enables improved performance for dataframe caching on the backend. If you decide *not* to add the full pair string in the feature string, FreqAI will operate in a reduced performance mode.
+
+Meanwhile, high level feature engineering is handled within `"feature_parameters":{}` in the FreqAI config. Within this file, it is possible to decide large scale feature expansions on top of the `base_features` such as "including correlated pairs" or "including informative timeframes" or even "including recent candles."
 
 It is advisable to start from the template `populate_any_indicators()` in the source provided example strategy (found in `templates/FreqaiExampleStrategy.py`) to ensure that the feature definitions are following the correct conventions. Here is an example of how to set the indicators and labels in the strategy:
 
@@ -15,7 +18,7 @@ It is advisable to start from the template `populate_any_indicators()` in the so
         """
         Function designed to automatically generate, name, and merge features
         from user-indicated timeframes in the configuration file. The user controls the indicators
-        passed to the training/prediction by prepending indicators with `'%-' + coin `
+        passed to the training/prediction by prepending indicators with `'%-' + pair `
         (see convention below). I.e., the user should not prepend any supporting metrics
         (e.g., bb_lowerband below) with % unless they explicitly want to pass that metric to the
         model.
@@ -23,37 +26,34 @@ It is advisable to start from the template `populate_any_indicators()` in the so
         :param df: strategy dataframe which will receive merges from informatives
         :param tf: timeframe of the dataframe which will modify the feature names
         :param informative: the dataframe associated with the informative pair
-        :param coin: the name of the coin which will modify the feature names.
         """
 
-        coin = pair.split('/')[0]
-
         if informative is None:
             informative = self.dp.get_pair_dataframe(pair, tf)
 
         # first loop is automatically duplicating indicators for time periods
         for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
             t = int(t)
-            informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
-            informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
-            informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t)
+            informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
+            informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
+            informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, window=t)
 
             bollinger = qtpylib.bollinger_bands(
                 qtpylib.typical_price(informative), window=t, stds=2.2
             )
-            informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"]
-            informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"]
-            informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"]
+            informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"]
+            informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"]
+            informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"]
 
-            informative[f"%-{coin}bb_width-period_{t}"] = (
-                informative[f"{coin}bb_upperband-period_{t}"]
-                - informative[f"{coin}bb_lowerband-period_{t}"]
-            ) / informative[f"{coin}bb_middleband-period_{t}"]
-            informative[f"%-{coin}close-bb_lower-period_{t}"] = (
-                informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"]
+            informative[f"%-{pair}bb_width-period_{t}"] = (
+                informative[f"{pair}bb_upperband-period_{t}"]
+                - informative[f"{pair}bb_lowerband-period_{t}"]
+            ) / informative[f"{pair}bb_middleband-period_{t}"]
+            informative[f"%-{pair}close-bb_lower-period_{t}"] = (
+                informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"]
             )
 
-            informative[f"%-{coin}relative_volume-period_{t}"] = (
+            informative[f"%-{pair}relative_volume-period_{t}"] = (
                 informative["volume"] / informative["volume"].rolling(t).mean()
             )
 
@@ -122,7 +122,7 @@ The `include_timeframes` in the config above are the timeframes (`tf`) of each c
 
 You can ask for each of the defined features to be included also for informative pairs using the `include_corr_pairlist`. This means that the feature set will include all the features from `populate_any_indicators` on all the `include_timeframes` for each of the correlated pairs defined in the config (`ETH/USD`, `LINK/USD`, and `BNB/USD` in the presented example).
 
-`include_shifted_candles` indicates the number of previous candles to include in the feature set. For example, `include_shifted_candles: 2` tells `FreqAI` to include the past 2 candles for each of the features in the feature set.
+`include_shifted_candles` indicates the number of previous candles to include in the feature set. For example, `include_shifted_candles: 2` tells FreqAI to include the past 2 candles for each of the features in the feature set.
 
 In total, the number of features the user of the presented example strat has created is: length of `include_timeframes` * no. features in `populate_any_indicators()` * length of `include_corr_pairlist` * no. `include_shifted_candles` * length of `indicator_periods_candles`
  $= 3 * 3 * 3 * 2 * 2 = 108$.
@@ -131,7 +131,7 @@ In total, the number of features the user of the presented example strat has cre
 
 Important metrics can be returned to the strategy at the end of each model training by assigning them to `dk.data['extra_returns_per_train']['my_new_value'] = XYZ` inside the custom prediction model class. 
 
-`FreqAI` takes the `my_new_value` assigned in this dictionary and expands it to fit the dataframe that is returned to the strategy. You can then use the returned metrics in your strategy through `dataframe['my_new_value']`. An example of how return values can be used in `FreqAI` are the `&*_mean` and `&*_std` values that are used to [created a dynamic target threshold](freqai-configuration.md#creating-a-dynamic-target-threshold).
+FreqAI takes the `my_new_value` assigned in this dictionary and expands it to fit the dataframe that is returned to the strategy. You can then use the returned metrics in your strategy through `dataframe['my_new_value']`. An example of how return values can be used in FreqAI are the `&*_mean` and `&*_std` values that are used to [created a dynamic target threshold](freqai-configuration.md#creating-a-dynamic-target-threshold).
 
 Another example, where the user wants to use live metrics from the trade database, is shown below:
 
@@ -141,15 +141,15 @@ Another example, where the user wants to use live metrics from the trade databas
     }
 ```
 
-You need to set the standard dictionary in the config so that `FreqAI` can return proper dataframe shapes. These values will likely be overridden by the prediction model, but in the case where the model has yet to set them, or needs a default initial value, the preset values are what will be returned.
+You need to set the standard dictionary in the config so that FreqAI can return proper dataframe shapes. These values will likely be overridden by the prediction model, but in the case where the model has yet to set them, or needs a default initial value, the pre-set values are what will be returned.
 
 ## Feature normalization
 
-`FreqAI` is strict when it comes to data normalization. The train features, $X^{train}$, are always normalized to [-1, 1] using a shifted min-max normalization:
+FreqAI is strict when it comes to data normalization. The train features, $X^{train}$, are always normalized to [-1, 1] using a shifted min-max normalization:
 
 $$X^{train}_{norm} = 2 * \frac{X^{train} - X^{train}.min()}{X^{train}.max() - X^{train}.min()} - 1$$
 
-All other data (test data and unseen prediction data in dry/live/backtest) is always automatically normalized to the training feature space according to industry standards. `FreqAI` stores all the metadata required to ensure that test and prediction features will be properly normalized and that predictions are properly denormalized. For this reason, it is not recommended to eschew industry standards and modify `FreqAI` internals - however - advanced users can do so by inheriting `train()` in their custom `IFreqaiModel` and using their own normalization functions.
+All other data (test data and unseen prediction data in dry/live/backtest) is always automatically normalized to the training feature space according to industry standards. FreqAI stores all the metadata required to ensure that test and prediction features will be properly normalized and that predictions are properly denormalized. For this reason, it is not recommended to eschew industry standards and modify FreqAI internals - however - advanced users can do so by inheriting `train()` in their custom `IFreqaiModel` and using their own normalization functions.
 
 ## Data dimensionality reduction with Principal Component Analysis
 
@@ -169,17 +169,17 @@ This will perform PCA on the features and reduce their dimensionality so that th
 
 The `inlier_metric` is a metric aimed at quantifying how similar a the features of a data point are to the most recent historic data points. 
 
-You define the lookback window by setting `inlier_metric_window` and `FreqAI` computes the distance between the present time point and each of the previous `inlier_metric_window` lookback points. A Weibull function is fit to each of the lookback distributions and its cumulative distribution function (CDF) is used to produce a quantile for each lookback point. The `inlier_metric` is then computed for each time point as the average of the corresponding lookback quantiles. The figure below explains the concept for an `inlier_metric_window` of 5.
+You define the lookback window by setting `inlier_metric_window` and FreqAI computes the distance between the present time point and each of the previous `inlier_metric_window` lookback points. A Weibull function is fit to each of the lookback distributions and its cumulative distribution function (CDF) is used to produce a quantile for each lookback point. The `inlier_metric` is then computed for each time point as the average of the corresponding lookback quantiles. The figure below explains the concept for an `inlier_metric_window` of 5.
 
 ![inlier-metric](assets/freqai_inlier-metric.jpg)
 
-`FreqAI` adds the `inlier_metric` to the training features and hence gives the model access to a novel type of temporal information. 
+FreqAI adds the `inlier_metric` to the training features and hence gives the model access to a novel type of temporal information. 
 
 This function does **not** remove outliers from the data set.
 
 ## Weighting features for temporal importance
 
-`FreqAI` allows you to set a `weight_factor` to weight recent data more strongly than past data via an exponential function:
+FreqAI allows you to set a `weight_factor` to weight recent data more strongly than past data via an exponential function:
 
 $$ W_i = \exp(\frac{-i}{\alpha*n}) $$
 
@@ -189,13 +189,13 @@ where $W_i$ is the weight of data point $i$ in a total set of $n$ data points. B
 
 ## Outlier detection
 
-Equity and crypto markets suffer from a high level of non-patterned noise in the form of outlier data points. `FreqAI` implements a variety of methods to identify such outliers and hence mitigate risk.
+Equity and crypto markets suffer from a high level of non-patterned noise in the form of outlier data points. FreqAI implements a variety of methods to identify such outliers and hence mitigate risk.
 
 ### Identifying outliers with the Dissimilarity Index (DI)
 
  The Dissimilarity Index (DI) aims to quantify the uncertainty associated with each prediction made by the model. 
 
-You can tell `FreqAI` to remove outlier data points from the training/test data sets using the DI by including the following statement in the config:
+You can tell FreqAI to remove outlier data points from the training/test data sets using the DI by including the following statement in the config:
 
 ```json
     "freqai": {
@@ -205,7 +205,7 @@ You can tell `FreqAI` to remove outlier data points from the training/test data
     }
 ```
 
- The DI allows predictions which are outliers (not existent in the model feature space) to be thrown out due to low levels of certainty. To do so, `FreqAI` measures the distance between each training data point (feature vector), $X_{a}$, and all other training data points:
+ The DI allows predictions which are outliers (not existent in the model feature space) to be thrown out due to low levels of certainty. To do so, FreqAI measures the distance between each training data point (feature vector), $X_{a}$, and all other training data points:
 
 $$ d_{ab} = \sqrt{\sum_{j=1}^p(X_{a,j}-X_{b,j})^2} $$
 
@@ -229,7 +229,7 @@ Below is a figure that describes the DI for a 3D data set.
 
 ### Identifying outliers using a Support Vector Machine (SVM)
 
-You can tell `FreqAI` to remove outlier data points from the training/test data sets using a Support Vector Machine (SVM) by including the following statement in the config:
+You can tell FreqAI to remove outlier data points from the training/test data sets using a Support Vector Machine (SVM) by including the following statement in the config:
 
 ```json
     "freqai": {
@@ -241,7 +241,7 @@ You can tell `FreqAI` to remove outlier data points from the training/test data
 
 The SVM will be trained on the training data and any data point that the SVM deems to be beyond the feature space will be removed.
 
-`FreqAI` uses `sklearn.linear_model.SGDOneClassSVM` (details are available on scikit-learn's webpage [here](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.SGDOneClassSVM.html) (external website)) and you can elect to provide additional parameters for the SVM, such as `shuffle`, and `nu`.
+FreqAI uses `sklearn.linear_model.SGDOneClassSVM` (details are available on scikit-learn's webpage [here](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.SGDOneClassSVM.html) (external website)) and you can elect to provide additional parameters for the SVM, such as `shuffle`, and `nu`.
 
 The parameter `shuffle` is by default set to `False` to ensure consistent results. If it is set to `True`, running the SVM multiple times on the same data set might result in different outcomes due to `max_iter` being to low for the algorithm to reach the demanded `tol`. Increasing `max_iter` solves this issue but causes the procedure to take longer time.
 
@@ -249,7 +249,7 @@ The parameter `nu`, *very* broadly, is the amount of data points that should be
 
 ### Identifying outliers with DBSCAN
 
-You can configure `FreqAI` to use DBSCAN to cluster and remove outliers from the training/test data set or incoming outliers from predictions, by activating `use_DBSCAN_to_remove_outliers` in the config:
+You can configure FreqAI to use DBSCAN to cluster and remove outliers from the training/test data set or incoming outliers from predictions, by activating `use_DBSCAN_to_remove_outliers` in the config:
 
 ```json
     "freqai": {
@@ -265,4 +265,4 @@ Given a number of data points $N$, and a distance $\varepsilon$, DBSCAN clusters
 
 ![dbscan](assets/freqai_dbscan.jpg)
 
-`FreqAI` uses `sklearn.cluster.DBSCAN` (details are available on scikit-learn's webpage [here](https://scikit-learn.org/stable/modules/generated/sklearn.cluster.DBSCAN.html) (external website)) with `min_samples` ($N$) taken as 1/4 of the no. of time points in the feature set. `eps` ($\varepsilon$) is computed automatically as the elbow point in the *k-distance graph* computed from the nearest neighbors in the pairwise distances of all data points in the feature set.
+FreqAI uses `sklearn.cluster.DBSCAN` (details are available on scikit-learn's webpage [here](https://scikit-learn.org/stable/modules/generated/sklearn.cluster.DBSCAN.html) (external website)) with `min_samples` ($N$) taken as 1/4 of the no. of time points (candles) in the feature set. `eps` ($\varepsilon$) is computed automatically as the elbow point in the *k-distance graph* computed from the nearest neighbors in the pairwise distances of all data points in the feature set.

docs/freqai-parameter-table.md

@@ -1,52 +1,53 @@
 # Parameter table
 
-The table below will list all configuration parameters available for `FreqAI`. Some of the parameters are exemplified in `config_examples/config_freqai.example.json`.
+The table below will list all configuration parameters available for FreqAI. Some of the parameters are exemplified in `config_examples/config_freqai.example.json`.
 
 Mandatory parameters are marked as **Required** and have to be set in one of the suggested ways.
 
 |  Parameter | Description |
 |------------|-------------|
 |  |  **General configuration parameters**
-| `freqai` | **Required.** <br> The parent dictionary containing all the parameters for controlling `FreqAI`. <br> **Datatype:** Dictionary.
+| `freqai` | **Required.** <br> The parent dictionary containing all the parameters for controlling FreqAI. <br> **Datatype:** Dictionary.
 | `train_period_days` | **Required.** <br> Number of days to use for the training data (width of the sliding window). <br> **Datatype:** Positive integer.
 | `backtest_period_days` | **Required.** <br> Number of days to inference from the trained model before sliding the `train_period_days` window defined above, and retraining the model during backtesting (more info [here](freqai-running.md#backtesting)). This can be fractional days, but beware that the provided `timerange` will be divided by this number to yield the number of trainings necessary to complete the backtest. <br> **Datatype:** Float.
 | `identifier` | **Required.** <br> A unique ID for the current model. If models are saved to disk, the `identifier` allows for reloading specific pre-trained models/data. <br> **Datatype:** String.
-| `live_retrain_hours` | Frequency of retraining during dry/live runs. <br> **Datatype:** Float > 0. <br> Default: 0 (models retrain as often as possible).
-| `expiration_hours` | Avoid making predictions if a model is more than `expiration_hours` old. <br> **Datatype:** Positive integer. <br> Default: 0 (models never expire).
+| `live_retrain_hours` | Frequency of retraining during dry/live runs. <br> **Datatype:** Float > 0. <br> Default: `0` (models retrain as often as possible).
+| `expiration_hours` | Avoid making predictions if a model is more than `expiration_hours` old. <br> **Datatype:** Positive integer. <br> Default: `0` (models never expire).
 | `purge_old_models` | Delete obsolete models. <br> **Datatype:** Boolean. <br> Default: `False` (all historic models remain on disk).
 | `save_backtest_models` | Save models to disk when running backtesting. Backtesting operates most efficiently by saving the prediction data and reusing them directly for subsequent runs (when you wish to tune entry/exit parameters). Saving backtesting models to disk also allows to use the same model files for starting a dry/live instance with the same model `identifier`. <br> **Datatype:** Boolean. <br> Default: `False` (no models are saved).
 | `fit_live_predictions_candles` | Number of historical candles to use for computing target (label) statistics from prediction data, instead of from the training dataset (more information can be found [here](freqai-configuration.md#creating-a-dynamic-target-threshold)). <br> **Datatype:** Positive integer.
 | `follow_mode` | Use a `follower` that will look for models associated with a specific `identifier` and load those for inferencing. A `follower` will **not** train new models. <br> **Datatype:** Boolean. <br> Default: `False`.
 | `continual_learning` | Use the final state of the most recently trained model as starting point for the new model, allowing for incremental learning (more information can be found [here](freqai-running.md#continual-learning)). <br> **Datatype:** Boolean. <br> Default: `False`.
+| `write_metrics_to_disk` | Collect train timings, inference timings and cpu usage in json file. <br> **Datatype:** Boolean. <br> Default: `False`
 |  |  **Feature parameters**
 | `feature_parameters` | A dictionary containing the parameters used to engineer the feature set. Details and examples are shown [here](freqai-feature-engineering.md). <br> **Datatype:** Dictionary.
 | `include_timeframes` | A list of timeframes that all indicators in `populate_any_indicators` will be created for. The list is added as features to the base indicators dataset. <br> **Datatype:** List of timeframes (strings).
-| `include_corr_pairlist` | A list of correlated coins that `FreqAI` will add as additional features to all `pair_whitelist` coins. All indicators set in `populate_any_indicators` during feature engineering (see details [here](freqai-feature-engineering.md)) will be created for each correlated coin. The correlated coins features are added to the base indicators dataset. <br> **Datatype:** List of assets (strings).
+| `include_corr_pairlist` | A list of correlated coins that FreqAI will add as additional features to all `pair_whitelist` coins. All indicators set in `populate_any_indicators` during feature engineering (see details [here](freqai-feature-engineering.md)) will be created for each correlated coin. The correlated coins features are added to the base indicators dataset. <br> **Datatype:** List of assets (strings).
 | `label_period_candles` | Number of candles into the future that the labels are created for. This is used in `populate_any_indicators` (see `templates/FreqaiExampleStrategy.py` for detailed usage). You can create custom labels and choose whether to make use of this parameter or not. <br> **Datatype:** Positive integer.
-| `include_shifted_candles` | Add features from previous candles to subsequent candles with the intent of adding historical information. If used, `FreqAI` will duplicate and shift all features from the `include_shifted_candles` previous candles so that the information is available for the subsequent candle. <br> **Datatype:** Positive integer.
+| `include_shifted_candles` | Add features from previous candles to subsequent candles with the intent of adding historical information. If used, FreqAI will duplicate and shift all features from the `include_shifted_candles` previous candles so that the information is available for the subsequent candle. <br> **Datatype:** Positive integer.
 | `weight_factor` | Weight training data points according to their recency (see details [here](freqai-feature-engineering.md#weighting-features-for-temporal-importance)). <br> **Datatype:** Positive float (typically < 1).
-| `indicator_max_period_candles` | **No longer used (#7325)**. Replaced by `startup_candle_count` which is set in the [strategy](freqai-configuration.md#building-a-freqai-strategy). `startup_candle_count` is timeframe independent and defines the maximum *period* used in `populate_any_indicators()` for indicator creation. `FreqAI` uses this parameter together with the maximum timeframe in `include_time_frames` to calculate how many data points to download such that the first data point does not include a NaN <br> **Datatype:** Positive integer.
+| `indicator_max_period_candles` | **No longer used (#7325)**. Replaced by `startup_candle_count` which is set in the [strategy](freqai-configuration.md#building-a-freqai-strategy). `startup_candle_count` is timeframe independent and defines the maximum *period* used in `populate_any_indicators()` for indicator creation. FreqAI uses this parameter together with the maximum timeframe in `include_time_frames` to calculate how many data points to download such that the first data point does not include a NaN. <br> **Datatype:** Positive integer.
 | `indicator_periods_candles` | Time periods to calculate indicators for. The indicators are added to the base indicator dataset. <br> **Datatype:** List of positive integers.
-| `stratify_training_data` | Split the feature set into training and testing datasets. For example, `stratify_training_data: 2` would set every 2nd data point into a separate dataset to be pulled from during training/testing. See details about how it works [here](freqai-running.md#data-stratification-for-training-and-testing-the-model). <br> **Datatype:** Positive integer.
-| `principal_component_analysis` | Automatically reduce the dimensionality of the data set using Principal Component Analysis. See details about how it works [here](#reducing-data-dimensionality-with-principal-component-analysis) <br> **Datatype:** Boolean. defaults to `false`.
-| `plot_feature_importances` | Create a feature importance plot for each model for the top/bottom `plot_feature_importances` number of features.<br> **Datatype:** Integer, defaults to `0`.
+| `principal_component_analysis` | Automatically reduce the dimensionality of the data set using Principal Component Analysis. See details about how it works [here](#reducing-data-dimensionality-with-principal-component-analysis) <br> **Datatype:** Boolean. <br> Default: `False`.
+| `plot_feature_importances` | Create a feature importance plot for each model for the top/bottom `plot_feature_importances` number of features. <br> **Datatype:** Integer. <br> Default: `0`.
 | `DI_threshold` | Activates the use of the Dissimilarity Index for outlier detection when set to > 0. See details about how it works [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di). <br> **Datatype:** Positive float (typically < 1).
 | `use_SVM_to_remove_outliers` | Train a support vector machine to detect and remove outliers from the training dataset, as well as from incoming data points. See details about how it works [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm). <br> **Datatype:** Boolean.
 | `svm_params` | All parameters available in Sklearn's `SGDOneClassSVM()`. See details about some select parameters [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm). <br> **Datatype:** Dictionary.
 | `use_DBSCAN_to_remove_outliers` | Cluster data using the DBSCAN algorithm to identify and remove outliers from training and prediction data. See details about how it works [here](freqai-feature-engineering.md#identifying-outliers-with-dbscan). <br> **Datatype:** Boolean. 
-| `inlier_metric_window` | If set, `FreqAI` adds an `inlier_metric` to the training feature set and set the lookback to be the `inlier_metric_window`, i.e., the number of previous time points to compare the current candle to. Details of how the `inlier_metric` is computed can be found [here](freqai-feature-engineering.md#inlier-metric). <br> **Datatype:** Integer. <br> Default: 0.
-| `noise_standard_deviation` | If set, `FreqAI` adds noise to the training features with the aim of preventing overfitting. `FreqAI` generates random deviates from a gaussian distribution with a standard deviation of `noise_standard_deviation` and adds them to all data points. `noise_standard_deviation` should be kept relative to the normalized space, i.e., between -1 and 1. In other words, since data in `FreqAI` is always normalized to be between -1 and 1, `noise_standard_deviation: 0.05` would result in 32% of the data being randomly increased/decreased by more than 2.5% (i.e., the percent of data falling within the first standard deviation). <br> **Datatype:** Integer. <br> Default: 0.
-| `outlier_protection_percentage` | Enable to prevent outlier detection methods from discarding too much data. If more than `outlier_protection_percentage` % of points are detected as outliers by the SVM or DBSCAN, `FreqAI` will log a warning message and ignore outlier detection, i.e., the original dataset will be kept intact. If the outlier protection is triggered, no predictions will be made based on the training dataset. <br> **Datatype:** Float. <br> Default: `30`.
+| `inlier_metric_window` | If set, FreqAI adds an `inlier_metric` to the training feature set and set the lookback to be the `inlier_metric_window`, i.e., the number of previous time points to compare the current candle to. Details of how the `inlier_metric` is computed can be found [here](freqai-feature-engineering.md#inlier-metric). <br> **Datatype:** Integer. <br> Default: `0`.
+| `noise_standard_deviation` | If set, FreqAI adds noise to the training features with the aim of preventing overfitting. FreqAI generates random deviates from a gaussian distribution with a standard deviation of `noise_standard_deviation` and adds them to all data points. `noise_standard_deviation` should be kept relative to the normalized space, i.e., between -1 and 1. In other words, since data in FreqAI is always normalized to be between -1 and 1, `noise_standard_deviation: 0.05` would result in 32% of the data being randomly increased/decreased by more than 2.5% (i.e., the percent of data falling within the first standard deviation). <br> **Datatype:** Integer. <br> Default: `0`.
+| `outlier_protection_percentage` | Enable to prevent outlier detection methods from discarding too much data. If more than `outlier_protection_percentage` % of points are detected as outliers by the SVM or DBSCAN, FreqAI will log a warning message and ignore outlier detection, i.e., the original dataset will be kept intact. If the outlier protection is triggered, no predictions will be made based on the training dataset. <br> **Datatype:** Float. <br> Default: `30`.
 | `reverse_train_test_order` | Split the feature dataset (see below) and use the latest data split for training and test on historical split of the data. This allows the model to be trained up to the most recent data point, while avoiding overfitting. However, you should be careful to understand the unorthodox nature of this parameter before employing it. <br> **Datatype:** Boolean. <br> Default: `False` (no reversal).
 |  |  **Data split parameters**
 | `data_split_parameters` | Include any additional parameters available from Scikit-learn `test_train_split()`, which are shown [here](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html) (external website). <br> **Datatype:** Dictionary.
 | `test_size` | The fraction of data that should be used for testing instead of training. <br> **Datatype:** Positive float < 1.
-| `shuffle` | Shuffle the training data points during training. Typically, for time-series forecasting, this is set to `False`. <br> **Datatype:** Boolean.
+| `shuffle` | Shuffle the training data points during training. Typically, to not remove the chronological order of data in time-series forecasting, this is set to `False`. <br> **Datatype:** Boolean. <br> Defaut: `False`.
 |  |  **Model training parameters**
-| `model_training_parameters` | A flexible dictionary that includes all parameters available by the selected model library. For example, if you use `LightGBMRegressor`, this dictionary can contain any parameter available by the `LightGBMRegressor` [here](https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMRegressor.html) (external website). If you select a different model, this dictionary can contain any parameter from that model.  <br> **Datatype:** Dictionary.
-| `n_estimators` | The number of boosted trees to fit in regression. <br> **Datatype:** Integer.
-| `learning_rate` | Boosting learning rate during regression. <br> **Datatype:** Float.
+| `model_training_parameters` | A flexible dictionary that includes all parameters available by the selected model library. For example, if you use `LightGBMRegressor`, this dictionary can contain any parameter available by the `LightGBMRegressor` [here](https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMRegressor.html) (external website). If you select a different model, this dictionary can contain any parameter from that model. A list of the currently available models can be found [here](freqai-configuration.md#using-different-prediction-models).  <br> **Datatype:** Dictionary.
+| `n_estimators` | The number of boosted trees to fit in the training of the model. <br> **Datatype:** Integer.
+| `learning_rate` | Boosting learning rate during training of the model. <br> **Datatype:** Float.
 | `n_jobs`, `thread_count`, `task_type` | Set the number of threads for parallel processing and the `task_type` (`gpu` or `cpu`). Different model libraries use different parameter names. <br> **Datatype:** Float.
 |  |  **Extraneous parameters**
 | `keras` | If the selected model makes use of Keras (typical for Tensorflow-based prediction models), this flag needs to be activated so that the model save/loading follows Keras standards. <br> **Datatype:** Boolean. <br> Default: `False`.
-| `conv_width` | The width of a convolutional neural network input tensor. This replaces the need for shifting candles (`include_shifted_candles`) by feeding in historical data points as the second dimension of the tensor. Technically, this parameter can also be used for regressors, but it only adds computational overhead and does not change the model training/prediction. <br> **Datatype:** Integer. <br> Default: 2.
+| `conv_width` | The width of a convolutional neural network input tensor. This replaces the need for shifting candles (`include_shifted_candles`) by feeding in historical data points as the second dimension of the tensor. Technically, this parameter can also be used for regressors, but it only adds computational overhead and does not change the model training/prediction. <br> **Datatype:** Integer. <br> Default: `2`.
+| `reduce_df_footprint` | Recast all numeric columns to float32/int32, with the objective of reducing ram/disk usage and decreasing train/inference timing. This parameter is set in the main level of the Freqtrade configuration file (not inside FreqAI). <br> **Datatype:** Boolean. <br> Default: `False`.

docs/freqai-running.md

@@ -1,6 +1,6 @@
 # Running FreqAI
 
-There are two ways to train and deploy an adaptive machine learning model - live deployment and historical backtesting. In both cases, `FreqAI` runs/simulates periodic retraining of models as shown in the following figure:
+There are two ways to train and deploy an adaptive machine learning model - live deployment and historical backtesting. In both cases, FreqAI runs/simulates periodic retraining of models as shown in the following figure:
 
 ![freqai-window](assets/freqai_moving-window.jpg)
 
@@ -33,7 +33,7 @@ FreqAI automatically downloads the proper amount of data needed to ensure traini
 
 ### Saving prediction data
 
-All predictions made during the lifetime of a specific `identifier` model are stored in `historical_predictions.pkl` to allow for reloading after a crash or changes made to the config.
+All predictions made during the lifetime of a specific `identifier` model are stored in `historic_predictions.pkl` to allow for reloading after a crash or changes made to the config.
 
 ### Purging old model data
 
@@ -73,21 +73,33 @@ Backtesting mode requires [downloading the necessary data](#downloading-data-to-
 
 To allow for tweaking your strategy (**not** the features!), FreqAI will automatically save the predictions during backtesting so that they can be reused for future backtests and live runs using the same `identifier` model. This provides a performance enhancement geared towards enabling **high-level hyperopting** of entry/exit criteria.
 
-An additional directory called `predictions`, which contains all the predictions stored in `hdf` format, will be created in the `unique-id` folder.
+An additional directory called `backtesting_predictions`, which contains all the predictions stored in `hdf` format, will be created in the `unique-id` folder.
 
-To change your **features**, you **must** set a new `identifier` in the config to signal to `FreqAI` to train new models.
+To change your **features**, you **must** set a new `identifier` in the config to signal to FreqAI to train new models.
 
 To save the models generated during a particular backtest so that you can start a live deployment from one of them instead of training a new model, you must set `save_backtest_models` to `True` in the config.
 
+### Backtest live models
+
+FreqAI allow you to reuse ready models through the backtest parameter `--freqai-backtest-live-models`. This can be useful when you want to reuse models generated in dry/run for comparison or other study. For that, you must set `"purge_old_models"` to `True` in the config.
+
+The `--timerange` parameter must not be informed, as it will be automatically calculated through the training end dates of the models.
+
+Each model has an identifier derived from the training end date. If you have only 1 model trained, FreqAI will backtest from the training end date until the current date. If you have more than 1 model, each model will perform the backtesting according to the training end date until the training end date of the next model and so on. For the last model, the period of the previous model will be used for the execution.
+
+!!! Note
+    Currently, there is no checking for expired models, even if the `expired_hours` parameter is set.
+
+
 ### Downloading data to cover the full backtest period
 
-For live/dry deployments, FreqAI will download the necessary data automatically. However, to use backtesting functionality, you need to download the necessary data using `download-data` (details [here](data-download.md#data-downloading)). You need to pay careful attention to understanding how much *additional* data needs to be downloaded to ensure that there is a sufficient amount of training data *before* the start of the backtesting timerange. The amount of additional data can be roughly estimated by moving the start date of the timerange backwards by `train_period_days` and the `startup_candle_count` (see the [parameter table](freqai-parameter-table.md) for detailed descriptions of these parameters) from the beginning of the desired backtesting timerange. 
+For live/dry deployments, FreqAI will download the necessary data automatically. However, to use backtesting functionality, you need to download the necessary data using `download-data` (details [here](data-download.md#data-downloading)). You need to pay careful attention to understanding how much *additional* data needs to be downloaded to ensure that there is a sufficient amount of training data *before* the start of the backtesting time range. The amount of additional data can be roughly estimated by moving the start date of the time range backwards by `train_period_days` and the `startup_candle_count` (see the [parameter table](freqai-parameter-table.md) for detailed descriptions of these parameters) from the beginning of the desired backtesting time range. 
 
-As an example, to backtest the `--timerange 20210501-20210701` using the [example config](freqai-configuration.md#setting-up-the-configuration-file) which sets `train_period_days` to 30, together with `startup_candle_count: 40` on a maximum `include_timeframes` of 1h, the start date for the downloaded data needs to be `20210501` - 30 days - 40 * 1h / 24 hours = 20210330 (31.7 days earlier than the start of the desired training timerange).
+As an example, to backtest the `--timerange 20210501-20210701` using the [example config](freqai-configuration.md#setting-up-the-configuration-file) which sets `train_period_days` to 30, together with `startup_candle_count: 40` on a maximum `include_timeframes` of 1h, the start date for the downloaded data needs to be `20210501` - 30 days - 40 * 1h / 24 hours = 20210330 (31.7 days earlier than the start of the desired training time range).
 
 ### Deciding the size of the sliding training window and backtesting duration
 
-The backtesting timerange is defined with the typical `--timerange` parameter in the configuration file. The duration of the sliding training window is set by `train_period_days`, whilst `backtest_period_days` is the sliding backtesting window, both in number of days (`backtest_period_days` can be
+The backtesting time range is defined with the typical `--timerange` parameter in the configuration file. The duration of the sliding training window is set by `train_period_days`, whilst `backtest_period_days` is the sliding backtesting window, both in number of days (`backtest_period_days` can be
 a float to indicate sub-daily retraining in live/dry mode). In the presented [example config](freqai-configuration.md#setting-up-the-configuration-file) (found in `config_examples/config_freqai.example.json`), the user is asking FreqAI to use a training period of 30 days and backtest on the subsequent 7 days. After the training of the model, FreqAI will backtest the subsequent 7 days. The "sliding window" then moves one week forward (emulating FreqAI retraining once per week in live mode) and the new model uses the previous 30 days (including the 7 days used for backtesting by the previous model) to train. This is repeated until the end of `--timerange`.  This means that if you set `--timerange 20210501-20210701`, FreqAI will have trained 8 separate models at the end of `--timerange` (because the full range comprises 8 weeks).
 
 !!! Note
@@ -105,23 +117,6 @@ During dry/live mode, FreqAI trains each coin pair sequentially (on separate thr
 
 In the presented example config, the user will only allow predictions on models that are less than 1/2 hours old.
 
-## Data stratification for training and testing the model
-
-You can stratify (group) the training/testing data using:
-
-```json
-    "freqai": {
-        "feature_parameters" : {
-            "stratify_training_data": 3
-        }
-    }
-```
-
-This will split the data chronologically so that every Xth data point is used to test the model after training. In the example above, the user is asking for every third data point in the dataframe to be used for
-testing; the other points are used for training.
-
-The test data is used to evaluate the performance of the model after training. If the test score is high, the model is able to capture the behavior of the data well. If the test score is low, either the model does not capture the complexity of the data, the test data is significantly different from the train data, or a different type of model should be used.
-
 ## Controlling the model learning process
 
 Model training parameters are unique to the selected machine learning library. FreqAI allows you to set any parameter for any library using the `model_training_parameters` dictionary in the config. The example config (found in `config_examples/config_freqai.example.json`) shows some of the example parameters associated with `Catboost` and `LightGBM`, but you can add any parameters available in those libraries or any other machine learning library you choose to implement.
@@ -132,7 +127,7 @@ The FreqAI specific parameter `label_period_candles` defines the offset (number
 
 ## Continual learning
 
-You can choose to adopt a continual learning scheme by setting `"continual_learning": true` in the config. By enabling `continual_learning`, after training an initial model from scratch, subsequent trainings will start from the final model state of the preceding training. This gives the new model a "memory" of the previous state. By default, this is set to `false` which means that all new models are trained from scratch, without input from previous models.
+You can choose to adopt a continual learning scheme by setting `"continual_learning": true` in the config. By enabling `continual_learning`, after training an initial model from scratch, subsequent trainings will start from the final model state of the preceding training. This gives the new model a "memory" of the previous state. By default, this is set to `False` which means that all new models are trained from scratch, without input from previous models.
 
 ## Hyperopt
 
@@ -159,15 +154,32 @@ dataframe['outlier'] = np.where(dataframe['DI_values'] > self.di_max.value/10, 1
 
 This specific hyperopt would help you understand the appropriate `DI_values` for your particular parameter space.
 
+## Using Tensorboard
+
+CatBoost models benefit from tracking training metrics via Tensorboard. You can take advantage of the FreqAI integration to track training and evaluation performance across all coins and across all retrainings. Tensorboard is activated via the following command:
+
+```bash
+cd freqtrade
+tensorboard --logdir user_data/models/unique-id
+```
+
+where `unique-id` is the `identifier` set in the `freqai` configuration file. This command must be run in a separate shell if you wish to view the output in your browser at 127.0.0.1:6060 (6060 is the default port used by Tensorboard).
+
+![tensorboard](assets/tensorboard.jpg)
+
 ## Setting up a follower
 
 You can indicate to the bot that it should not train models, but instead should look for models trained by a leader with a specific `identifier` by defining:
 
 ```json
     "freqai": {
+        "enabled": true,
         "follow_mode": true,
-        "identifier": "example"
+        "identifier": "example",
+        "feature_parameters": {
+        // leader bots feature_parameters inserted here 
+        },
     }
 ```
 
-In this example, the user has a leader bot with the `"identifier": "example"`. The leader bot is already running or is launched simultaneously with the follower. The follower will load models created by the leader and inference them to obtain predictions instead of training its own models.
+In this example, the user has a leader bot with the `"identifier": "example"`. The leader bot is already running or is launched simultaneously with the follower. The follower will load models created by the leader and inference them to obtain predictions instead of training its own models. The user will also need to duplicate the `feature_parameters` parameters from from the leaders freqai configuration file into the freqai section of the followers config. 

docs/freqai.md

@@ -1,10 +1,10 @@
 ![freqai-logo](assets/freqai_doc_logo.svg)
 
-# `FreqAI`
+# FreqAI
 
 ## Introduction
 
-`FreqAI` is a software designed to automate a variety of tasks associated with training a predictive machine learning model to generate market forecasts given a set of input features.
+FreqAI is a software designed to automate a variety of tasks associated with training a predictive machine learning model to generate market forecasts given a set of input signals. In general, the FreqAI aims to be a sand-box for easily deploying robust machine-learning libraries on real-time data ([details])(#freqai-position-in-open-source-machine-learning-landscape).
 
 Features include:
 
@@ -23,7 +23,7 @@ Features include:
 
 ## Quick start
 
-The easiest way to quickly test `FreqAI` is to run it in dry mode with the following command:
+The easiest way to quickly test FreqAI is to run it in dry mode with the following command:
 
 ```bash
 freqtrade trade --config config_examples/config_freqai.example.json --strategy FreqaiExampleStrategy --freqaimodel LightGBMRegressor --strategy-path freqtrade/templates
@@ -37,7 +37,7 @@ An example strategy, prediction model, and config to use as a starting points ca
 
 ## General approach
 
-You provide `FreqAI` with a set of custom *base indicators* (the same way as in a [typical Freqtrade strategy](strategy-customization.md)) as well as target values (*labels*). For each pair in the whitelist, `FreqAI` trains a model to predict the target values based on the input of custom indicators. The models are then consistently retrained, with a predetermined frequency, to adapt to market conditions. `FreqAI` offers the ability to both backtest strategies (emulating reality with periodic retraining on historic data) and deploy dry/live runs. In dry/live conditions, `FreqAI` can be set to constant retraining in a background thread to keep models as up to date as possible.
+You provide FreqAI with a set of custom *base indicators* (the same way as in a [typical Freqtrade strategy](strategy-customization.md)) as well as target values (*labels*). For each pair in the whitelist, FreqAI trains a model to predict the target values based on the input of custom indicators. The models are then consistently retrained, with a predetermined frequency, to adapt to market conditions. FreqAI offers the ability to both backtest strategies (emulating reality with periodic retraining on historic data) and deploy dry/live runs. In dry/live conditions, FreqAI can be set to constant retraining in a background thread to keep models as up to date as possible.
 
 An overview of the algorithm, explaining the data processing pipeline and model usage, is shown below.
 
@@ -45,21 +45,21 @@ An overview of the algorithm, explaining the data processing pipeline and model
 
 ### Important machine learning vocabulary
 
-**Features** - the parameters, based on historic data, on which a model is trained. All features for a single candle is stored as a vector. In `FreqAI`, you build a feature data sets from anything you can construct in the strategy.
+**Features** - the parameters, based on historic data, on which a model is trained. All features for a single candle are stored as a vector. In FreqAI, you build a feature data set from anything you can construct in the strategy.
 
-**Labels** - the target values that a model is trained toward. Each feature vector is associated with a single label that is defined by you within the strategy. These labels intentionally look into the future, and are not available to the model during dry/live/backtesting.
+**Labels** - the target values that the model is trained toward. Each feature vector is associated with a single label that is defined by you within the strategy. These labels intentionally look into the future and are what you are training the model to be able to predict.
 
-**Training** - the process of "teaching" the model to match the feature sets to the associated labels. Different types of models "learn" in different ways. More information about the different models can be found [here](freqai-configuration.md#using-different-prediction-models).
+**Training** - the process of "teaching" the model to match the feature sets to the associated labels. Different types of models "learn" in different ways which means that one might be better than another for a specific application. More information about the different models that are already implemented in FreqAI can be found [here](freqai-configuration.md#using-different-prediction-models).
 
-**Train data** - a subset of the feature data set that is fed to the model during training. This data directly influences weight connections in the model.
+**Train data** - a subset of the feature data set that is fed to the model during training to "teach" the model how to predict the targets. This data directly influences weight connections in the model.
 
 **Test data** - a subset of the feature data set that is used to evaluate the performance of the model after training. This data does not influence nodal weights within the model.
 
-**Inferencing** - the process of feeding a trained model new data on which it will make a prediction.
+**Inferencing** - the process of feeding a trained model new unseen data on which it will make a prediction. 
 
 ## Install prerequisites
 
-The normal Freqtrade install process will ask if you wish to install `FreqAI` dependencies. You should reply "yes" to this question if you wish to use `FreqAI`. If you did not reply yes, you can manually install these dependencies after the install with:
+The normal Freqtrade install process will ask if you wish to install FreqAI dependencies. You should reply "yes" to this question if you wish to use FreqAI. If you did not reply yes, you can manually install these dependencies after the install with:
 
 ``` bash
 pip install -r requirements-freqai.txt
@@ -70,18 +70,23 @@ pip install -r requirements-freqai.txt
 
 ### Usage with docker
 
-If you are using docker, a dedicated tag with `FreqAI` dependencies is available as `:freqai`. As such - you can replace the image line in your docker-compose file with `image: freqtradeorg/freqtrade:develop_freqai`. This image contains the regular `FreqAI` dependencies. Similar to native installs, Catboost will not be available on ARM based devices.
+If you are using docker, a dedicated tag with FreqAI dependencies is available as `:freqai`. As such - you can replace the image line in your docker-compose file with `image: freqtradeorg/freqtrade:develop_freqai`. This image contains the regular FreqAI dependencies. Similar to native installs, Catboost will not be available on ARM based devices.
+
+
+### FreqAI position in open-source machine learning landscape
+
+Forecasting chaotic time-series based systems, such as equity/cryptocurrency markets, requires a broad set of tools geared toward testing a wide range of hypotheses. Fortunately, a recent maturation of robust machine learning libraries (e.g. `scikit-learn`) has opened up a wide range of research possibilities. Scientists from a diverse range of fields can now easily prototype their studies on an abundance of established machine learning algorithms. Similarly, these user-friendly libraries enable "citzen scientists" to use their basic Python skills for data-exploration. However, leveraging these machine learning libraries on historical and live chaotic data sources can be logistically difficult and expensive. Additionally, robust data-collection, storage, and handling presents a disparate challenge. [`FreqAI`](#freqai) aims to provide a generalized and extensible open-sourced framework geared toward live deployments of adaptive modeling for market forecasting. The `FreqAI` framework is effectively a sandbox for the rich world of open-source machine learning libraries. Inside the `FreqAI` sandbox, users find they can combine a wide variety of third-party libraries to test creative hypotheses on a free live 24/7 chaotic data source - cryptocurrency exchange data. 
 
 ## Common pitfalls
 
-`FreqAI` cannot be combined with dynamic `VolumePairlists` (or any pairlist filter that adds and removes pairs dynamically).
-This is for performance reasons - `FreqAI` relies on making quick predictions/retrains. To do this effectively,
-it needs to download all the training data at the beginning of a dry/live instance. `FreqAI` stores and appends
-new candles automatically for future retrains. This means that if new pairs arrive later in the dry run due to a volume pairlist, it will not have the data ready. However, `FreqAI` does work with the `ShufflePairlist` or a `VolumePairlist` which keeps the total pairlist constant (but reorders the pairs according to volume).
+FreqAI cannot be combined with dynamic `VolumePairlists` (or any pairlist filter that adds and removes pairs dynamically).
+This is for performance reasons - FreqAI relies on making quick predictions/retrains. To do this effectively,
+it needs to download all the training data at the beginning of a dry/live instance. FreqAI stores and appends
+new candles automatically for future retrains. This means that if new pairs arrive later in the dry run due to a volume pairlist, it will not have the data ready. However, FreqAI does work with the `ShufflePairlist` or a `VolumePairlist` which keeps the total pairlist constant (but reorders the pairs according to volume).
 
 ## Credits
 
-`FreqAI` is developed by a group of individuals who all contribute specific skillsets to the project.
+FreqAI is developed by a group of individuals who all contribute specific skillsets to the project.
 
 Conception and software development:
 Robert Caulk @robcaulk
@@ -96,5 +101,4 @@ Software development:
 Wagner Costa @wagnercosta
 
 Beta testing and bug reporting:
-Stefan Gehring @bloodhunter4rc, @longyu, Andrew Robert Lawless @paranoidandy, Pascal Schmidt @smidelis, Ryan McMullan @smarmau,
-Juha Nykänen @suikula, Johan van der Vlugt @jooopiert, Richárd Józsa @richardjosza
+Stefan Gehring @bloodhunter4rc, @longyu, Andrew Lawless @paranoidandy, Pascal Schmidt @smidelis, Ryan McMullan @smarmau, Juha Nykänen @suikula, Johan van der Vlugt @jooopiert, Richárd Józsa @richardjosza, Timothy Pogue @wizrds

docs/includes/pairlists.md

@@ -22,6 +22,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
 
 * [`StaticPairList`](#static-pair-list) (default, if not configured differently)
 * [`VolumePairList`](#volume-pair-list)
+* [`ProducerPairList`](#producerpairlist)
 * [`AgeFilter`](#agefilter)
 * [`OffsetFilter`](#offsetfilter)
 * [`PerformanceFilter`](#performancefilter)
@@ -84,7 +85,7 @@ Filtering instances (not the first position in the list) will not apply any cach
 
 You can define a minimum volume with `min_value` - which will filter out pairs with a volume lower than the specified value in the specified timerange.
 
-### VolumePairList Advanced mode
+##### VolumePairList Advanced mode
 
 `VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a  more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
 
@@ -146,6 +147,32 @@ More sophisticated approach can be used, by using `lookback_timeframe` for candl
 !!! Note
     `VolumePairList` does not support backtesting mode.
 
+#### ProducerPairList
+
+With `ProducerPairList`, you can reuse the pairlist from a [Producer](producer-consumer.md) without explicitly defining the pairlist on each consumer.
+
+[Consumer mode](producer-consumer.md) is required for this pairlist to work.
+
+The pairlist will perform a check on active pairs against the current exchange configuration to avoid attempting to trade on invalid markets.
+
+You can limit the length of the pairlist with the optional parameter `number_assets`. Using `"number_assets"=0` or omitting this key will result in the reuse of all producer pairs valid for the current setup.
+
+```json
+"pairlists": [
+    {
+        "method": "ProducerPairList",
+        "number_assets": 5,
+        "producer_name": "default",
+    }
+],
+```
+
+
+!!! Tip "Combining pairlists"
+    This pairlist can be combined with all other pairlists and filters for further pairlist reduction, and can also act as an "additional" pairlist, on top of already defined pairs.
+    `ProducerPairList` can also be used multiple times in sequence, combining the pairs from multiple producers.
+    Obviously in complex such configurations, the Producer may not provide data for all pairs, so the strategy must be fit for this.
+
 #### AgeFilter
 
 Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity).
@@ -241,7 +268,7 @@ This option is disabled by default, and will only apply if set to > 0.
 The `max_value` setting removes pairs where the minimum value change is above a specified value.
 This is useful when an exchange has unbalanced limits. For example, if step-size = 1 (so you can only buy 1, or 2, or 3, but not 1.1 Coins) - and the price is pretty high (like 20\$) as the coin has risen sharply since the last limit adaption.
 As a result of the above, you can only buy for 20\$, or 40\$ - but not for 25\$.
-On exchanges that deduct fees from the receiving currency (e.g. FTX) - this can result in high value coins / amounts that are unsellable as the amount is slightly below the limit.
+On exchanges that deduct fees from the receiving currency (e.g. binance) - this can result in high value coins / amounts that are unsellable as the amount is slightly below the limit.
 
 The `low_price_ratio` setting removes pairs where a raise of 1 price unit (pip) is above the `low_price_ratio` ratio.
 This option is disabled by default, and will only apply if set to > 0.
@@ -259,6 +286,18 @@ Min price precision for SHITCOIN/BTC is 8 decimals. If its price is 0.00000011 -
 
 Shuffles (randomizes) pairs in the pairlist. It can be used for preventing the bot from trading some of the pairs more frequently then others when you want all pairs be treated with the same priority.
 
+By default, ShuffleFilter will shuffle pairs once per candle.
+To shuffle on every iteration, set `"shuffle_frequency"` to `"iteration"` instead of  the default of `"candle"`.
+
+``` json
+    {
+        "method": "ShuffleFilter", 
+        "shuffle_frequency": "candle",
+        "seed": 42
+    }
+
+```
+
 !!! Tip
     You may set the `seed` value for this Pairlist to obtain reproducible results, which can be useful for repeated backtesting sessions. If `seed` is not set, the pairs are shuffled in the non-repeatable random order. ShuffleFilter will automatically detect runmodes and apply the `seed` only for backtesting modes - if a `seed` value is set.
 

docs/index.md

@@ -32,7 +32,7 @@ Freqtrade is a free and open source crypto trading bot written in Python. It is
 - Run: Test your strategy with simulated money (Dry-Run mode) or deploy it with real money (Live-Trade mode).
 - Run using Edge (optional module): The concept is to find the best historical [trade expectancy](edge.md#expectancy) by markets based on variation of the stop-loss and then allow/reject markets to trade. The sizing of the trade is based on a risk of a percentage of your capital.
 - Control/Monitor: Use Telegram or a WebUI (start/stop the bot, show profit/loss, daily summary, current open trades results, etc.).
-- Analyse: Further analysis can be performed on either Backtesting data or Freqtrade trading history (SQL database), including automated standard plots, and methods to load the data into [interactive environments](data-analysis.md).
+- Analyze: Further analysis can be performed on either Backtesting data or Freqtrade trading history (SQL database), including automated standard plots, and methods to load the data into [interactive environments](data-analysis.md).
 
 ## Supported exchange marketplaces
 
@@ -40,7 +40,6 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual,
 
 - [X] [Binance](https://www.binance.com/)
 - [X] [Bittrex](https://bittrex.com/)
-- [X] [FTX](https://ftx.com/#a=2258149)
 - [X] [Gate.io](https://www.gate.io/ref/6266643)
 - [X] [Huobi](http://huobi.com/)
 - [X] [Kraken](https://kraken.com/)
@@ -51,7 +50,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual,
 
 - [X] [Binance](https://www.binance.com/)
 - [X] [Gate.io](https://www.gate.io/ref/6266643)
-- [X] [OKX](https://okx.com/).
+- [X] [OKX](https://okx.com/)
 
 Please make sure to read the [exchange specific notes](exchanges.md), as well as the [trading with leverage](leverage.md) documentation before diving in.
 

docs/producer-consumer.md

@@ -21,6 +21,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect
                 "name": "default", // This can be any name you'd like, default is "default"
                 "host": "127.0.0.1", // The host from your producer's api_server config
                 "port": 8080, // The port from your producer's api_server config
+                "secure": false, // Use a secure websockets connection, default false
                 "ws_token": "sercet_Ws_t0ken" // The ws_token from your producer's api_server config
             }
         ],
@@ -42,6 +43,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect
 | `producers.name` | **Required.** Name of this producer. This name must be used in calls to `get_producer_pairs()` and `get_producer_df()` if more than one producer is used.<br> **Datatype:** string
 | `producers.host` | **Required.** The hostname or IP address from your producer.<br> **Datatype:** string
 | `producers.port` | **Required.** The port matching the above host.<br> **Datatype:** string
+| `producers.secure` | **Optional.**  Use ssl in websockets connection. Default False.<br> **Datatype:** string
 | `producers.ws_token` | **Required.**  `ws_token` as configured on the producer.<br> **Datatype:** string
 | | **Optional settings**
 | `wait_timeout` | Timeout until we ping again if no message is received. <br>*Defaults to `300`.*<br> **Datatype:** Integer - in seconds.

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.3.7
-mkdocs==1.3.1
-mkdocs-material==8.5.3
+mkdocs==1.4.2
+mkdocs-material==8.5.10
 mdx_truly_sane_lists==1.3
-pymdown-extensions==9.5
+pymdown-extensions==9.8
 jinja2==3.1.2

docs/rest-api.md

@@ -389,6 +389,44 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th
 }
 ```
 
+#### Reverse Proxy setup
+
+When using [Nginx](https://nginx.org/en/docs/), the following configuration is required for WebSockets to work (Note this configuration is incomplete, it's missing some information and can not be used as is):
+
+Please make sure to replace `<freqtrade_listen_ip>` (and the subsequent port) with the IP and Port matching your configuration/setup.
+
+```
+http {
+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+        '' close;
+    }
+
+    #...
+
+    server {
+        #...
+
+        location / {
+            proxy_http_version 1.1;
+            proxy_pass http://<freqtrade_listen_ip>:8080;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection $connection_upgrade;
+            proxy_set_header Host $host;
+        }
+    }
+}
+```
+
+To properly configure your reverse proxy (securely), please consult it's documentation for proxying websockets.
+
+- **Traefik**: Traefik supports websockets out of the box, see the [documentation](https://doc.traefik.io/traefik/)
+- **Caddy**: Caddy v2 supports websockets out of the box, see the [documentation](https://caddyserver.com/docs/v2-upgrade#proxy)
+
+!!! Tip "SSL certificates"
+    You can use tools like certbot to setup ssl certificates to access your bot's UI through encrypted connection by using any fo the above reverse proxies.
+    While this will protect your data in transit, we do not recommend to run the freqtrade API outside of your private network (VPN, SSH tunnel).
+
 ### OpenAPI interface
 
 To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration.

docs/stoploss.md

@@ -24,7 +24,7 @@ These modes can be configured with these values:
 ```
 
 !!! Note
-    Stoploss on exchange is only supported for Binance (stop-loss-limit), Huobi (stop-limit), Kraken (stop-loss-market, stop-loss-limit), FTX (stop limit and stop-market) Gateio (stop-limit), and Kucoin (stop-limit and stop-market) as of now.  
+    Stoploss on exchange is only supported for Binance (stop-loss-limit), Huobi (stop-limit), Kraken (stop-loss-market, stop-loss-limit), Gateio (stop-limit), and Kucoin (stop-limit and stop-market) as of now.  
     <ins>Do not set too low/tight stoploss value if using stop loss on exchange!</ins>  
     If set to low/tight then you have greater risk of missing fill on the order and stoploss will not work.
 
@@ -87,7 +87,7 @@ At this stage the bot contains the following stoploss support modes:
 2. Trailing stop loss.
 3. Trailing stop loss, custom positive loss.
 4. Trailing stop loss only once the trade has reached a certain offset.
-5. [Custom stoploss function](strategy-advanced.md#custom-stoploss)
+5. [Custom stoploss function](strategy-callbacks.md#custom-stoploss)
 
 ### Static Stop Loss
 

docs/strategy-callbacks.md

@@ -159,6 +159,7 @@ The stoploss price can only ever move upwards - if the stoploss value returned f
 
 The method must return a stoploss value (float / number) as a percentage of the current price.
 E.g. If the `current_rate` is 200 USD, then returning `0.02` will set the stoploss price 2% lower, at 196 USD.
+During backtesting, `current_rate` (and `current_profit`) are provided against the candle's high (or low for short trades) - while the resulting stoploss is evaluated against the candle's low (or high for short trades).
 
 The absolute value of the return value is used (the sign is ignored), so returning `0.05` or `-0.05` have the same result, a stoploss 5% below the current price.
 
@@ -643,7 +644,7 @@ This callback is **not** called when there is an open order (either buy or sell)
 
 Additional Buys are ignored once you have reached the maximum amount of extra buys that you have set on `max_entry_position_adjustment`, but the callback is called anyway looking for partial exits.
 
-Position adjustments will always be applied in the direction of the trade, so a positive value will always increase your position (negative values will decrease your position), no matter if it's a long or short trade. Modifications to leverage are not possible.
+Position adjustments will always be applied in the direction of the trade, so a positive value will always increase your position (negative values will decrease your position), no matter if it's a long or short trade. Modifications to leverage are not possible, and the stake-amount is assumed to be before applying leverage.
 
 !!! Note "About stake size"
     Using fixed stake size means it will be the amount used for the first order, just like without position adjustment.

docs/strategy-customization.md

@@ -446,15 +446,17 @@ A full sample can be found [in the DataProvider section](#complete-data-provider
 
 ??? Note "Alternative candle types"
     Informative_pairs can also provide a 3rd tuple element defining the candle type explicitly.
-    Availability of alternative candle-types will depend on the trading-mode and the exchange. Details about this can be found in the exchange documentation.
+    Availability of alternative candle-types will depend on the trading-mode and the exchange. 
+    In general, spot pairs cannot be used in futures markets, and futures candles can't be used as informative pairs for spot bots.
+    Details about this may vary, if they do, this can be found in the exchange documentation.
 
     ``` python
     def informative_pairs(self):
         return [
-            ("ETH/USDT", "5m", ""),   # Uses default candletype, depends on trading_mode 
-            ("ETH/USDT", "5m", "spot"),   # Forces usage of spot candles
-            ("BTC/TUSD", "15m", "futures"),  # Uses futures candles
-            ("BTC/TUSD", "15m", "mark"),  # Uses mark candles
+            ("ETH/USDT", "5m", ""),   # Uses default candletype, depends on trading_mode (recommended)
+            ("ETH/USDT", "5m", "spot"),   # Forces usage of spot candles (only valid for bots running on spot markets).
+            ("BTC/TUSD", "15m", "futures"),  # Uses futures candles (only bots with `trading_mode=futures`)
+            ("BTC/TUSD", "15m", "mark"),  # Uses mark candles (only bots with `trading_mode=futures`)
         ]
     ```
 ***
@@ -655,13 +657,13 @@ This is where calling `self.dp.current_whitelist()` comes in handy.
 # fetch live / historical candle (OHLCV) data for the first informative pair
 inf_pair, inf_timeframe = self.informative_pairs()[0]
 informative = self.dp.get_pair_dataframe(pair=inf_pair,
-                                            timeframe=inf_timeframe)
+                                         timeframe=inf_timeframe)
 ```
 
 !!! Warning "Warning about backtesting"
-    Be careful when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()`
-    for the backtesting runmode) provides the full time-range in one go,
-    so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode.
+    In backtesting, `dp.get_pair_dataframe()` behavior differs depending on where it's called.
+    Within `populate_*()` methods, `dp.get_pair_dataframe()` returns the full timerange. Please make sure to not "look into the future" to avoid surprises when running in dry/live mode.
+    Within [callbacks](strategy-callbacks.md), you'll get the full timerange up to the current (simulated) candle.
 
 ### *get_analyzed_dataframe(pair, timeframe)*
 
@@ -670,13 +672,13 @@ It can also be used in specific callbacks to get the signal that caused the acti
 
 ``` python
 # fetch current dataframe
-if self.dp.runmode.value in ('live', 'dry_run'):
-    dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=metadata['pair'],
-                                                                timeframe=self.timeframe)
+dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=metadata['pair'],
+                                                         timeframe=self.timeframe)
 ```
 
 !!! Note "No data available"
     Returns an empty dataframe if the requested pair was not cached.
+    You can check for this with `if dataframe.empty:` and handle this case accordingly.
     This should not happen when using whitelisted pairs.
 
 ### *orderbook(pair, maximum)*
@@ -723,7 +725,7 @@ if self.dp.runmode.value in ('live', 'dry_run'):
 
 !!! Warning
     Although the ticker data structure is a part of the ccxt Unified Interface, the values returned by this method can
-    vary for different exchanges. For instance, many exchanges do not return `vwap` values, the FTX exchange
+    vary for different exchanges. For instance, many exchanges do not return `vwap` values, some exchanges
     does not always fills in the `last` field (so it can be None), etc. So you need to carefully verify the ticker
     data returned from the exchange and add appropriate error handling / defaults.
 

docs/strategy_migration.md

@@ -43,19 +43,25 @@ Note : `forcesell`, `forcebuy`, `emergencysell` are changed to `force_exit`, `fo
   * `order_time_in_force` buy -> entry, sell -> exit.
   * `order_types` buy -> entry, sell -> exit.
   * `unfilledtimeout` buy -> entry, sell -> exit.
+  * `ignore_buying_expired_candle_after` -> moved to root level instead of "ask_strategy/exit_pricing"
 * Terminology changes
   * Sell reasons changed to reflect the new naming of "exit" instead of sells. Be careful in your strategy if you're using `exit_reason` checks and eventually update your strategy.
     * `sell_signal` -> `exit_signal`
     * `custom_sell` -> `custom_exit`
     * `force_sell` -> `force_exit`
     * `emergency_sell` -> `emergency_exit`
+  * Order pricing
+    * `bid_strategy` -> `entry_pricing`
+    * `ask_strategy` -> `exit_pricing`
+    * `ask_last_balance` -> `price_last_balance`
+    * `bid_last_balance` -> `price_last_balance`
   * Webhook terminology changed from "sell" to "exit", and from "buy" to entry
-    * `webhookbuy` -> `webhookentry`
-    * `webhookbuyfill` -> `webhookentryfill`
-    * `webhookbuycancel` -> `webhookentrycancel`
-    * `webhooksell` -> `webhookexit`
-    * `webhooksellfill` -> `webhookexitfill`
-    * `webhooksellcancel` -> `webhookexitcancel`
+    * `webhookbuy` -> `entry`
+    * `webhookbuyfill` -> `entry_fill`
+    * `webhookbuycancel` -> `entry_cancel`
+    * `webhooksell` -> `exit`
+    * `webhooksellfill` -> `exit_fill`
+    * `webhooksellcancel` -> `exit_cancel`
   * Telegram notification settings
     * `buy` -> `entry`
     * `buy_fill` -> `entry_fill`
@@ -443,6 +449,7 @@ Please refer to the [pricing documentation](configuration.md#prices-used-for-ord
         "use_order_book": true,
         "order_book_top": 1,
         "bid_last_balance": 0.0
+        "ignore_buying_expired_candle_after": 120
     }
 }
 ```
@@ -466,6 +473,7 @@ after:
         "use_order_book": true,
         "order_book_top": 1,
         "price_last_balance": 0.0
-    }
+    },
+    "ignore_buying_expired_candle_after": 120
 }
 ```

docs/telegram-usage.md

@@ -77,6 +77,7 @@ Example configuration showing the different settings:
     "enabled": true,
     "token": "your_telegram_token",
     "chat_id": "your_telegram_chat_id",
+    "allow_custom_messages": true,
     "notification_settings": {
         "status": "silent",
         "warning": "on",
@@ -115,6 +116,7 @@ Example configuration showing the different settings:
 `show_candle` - show candle values as part of entry/exit messages. Only possible values are `"ohlc"` or `"off"`.
 
 `balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown.
+`allow_custom_messages` completely disable strategy messages.
 `reload` allows you to disable reload-buttons on selected messages.
 
 ## Create a custom keyboard (command shortcut buttons)

docs/updating.md

@@ -37,3 +37,12 @@ pip install -e .
 # Ensure freqUI is at the latest version
 freqtrade install-ui 
 ```
+
+### Problems updating
+
+Update-problems usually come missing dependencies (you didn't follow the above instructions) - or from updated dependencies, which fail to install (for example TA-lib).
+Please refer to the corresponding installation sections (common problems linked below)
+
+Common problems and their solutions:
+
+* [ta-lib update on windows](windows_installation.md#2-install-ta-lib)

docs/utils.md

@@ -169,6 +169,43 @@ Example: Search dedicated strategy path.
 freqtrade list-strategies --strategy-path ~/.freqtrade/strategies/
 ```
 
+## List freqAI models
+
+Use the `list-freqaimodels` subcommand to see all freqAI models available.
+
+This subcommand is useful for finding problems in your environment with loading freqAI models: modules with models that contain errors and failed to load are printed in red (LOAD FAILED), while models with duplicate names are printed in yellow (DUPLICATE NAME).
+
+```
+usage: freqtrade list-freqaimodels [-h] [-v] [--logfile FILE] [-V] [-c PATH]
+                                   [-d PATH] [--userdir PATH]
+                                   [--freqaimodel-path PATH] [-1] [--no-color]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --freqaimodel-path PATH
+                        Specify additional lookup path for freqaimodels.
+  -1, --one-column      Print output in one column.
+  --no-color            Disable colorization of hyperopt results. May be
+                        useful if you are redirecting output to a file.
+
+Common arguments:
+  -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
+  --logfile FILE        Log to the file specified. Special values are:
+                        'syslog', 'journald'. See the documentation for more
+                        details.
+  -V, --version         show program's version number and exit
+  -c PATH, --config PATH
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
+  -d PATH, --datadir PATH, --data-dir PATH
+                        Path to directory with historical backtesting data.
+  --userdir PATH, --user-data-dir PATH
+                        Path to userdata directory.
+
+```
+
 ## List Exchanges
 
 Use the `list-exchanges` subcommand to see the exchanges available for the bot.
@@ -226,7 +263,6 @@ equos            True     missing opt: fetchTicker, fetchTickers
 eterbase         True
 fcoin            True     missing opt: fetchMyTrades, fetchTickers
 fcoinjp          True     missing opt: fetchMyTrades, fetchTickers
-ftx              True
 gateio           True
 gemini           True
 gopax            True
@@ -332,7 +368,6 @@ fcoin               True     missing opt: fetchMyTrades, fetchTickers
 fcoinjp             True     missing opt: fetchMyTrades, fetchTickers
 flowbtc             False    missing: fetchOrder, fetchOHLCV
 foxbit              False    missing: fetchOrder, fetchOHLCV
-ftx                 True
 gateio              True
 gemini              True
 gopax               True

docs/webhook-config.md

@@ -10,37 +10,37 @@ Sample configuration (tested using IFTTT).
   "webhook": {
         "enabled": true,
         "url": "https://maker.ifttt.com/trigger/<YOUREVENT>/with/key/<YOURKEY>/",
-        "webhookentry": {
+        "entry": {
             "value1": "Buying {pair}",
             "value2": "limit {limit:8f}",
             "value3": "{stake_amount:8f} {stake_currency}"
         },
-        "webhookentrycancel": {
+        "entry_cancel": {
             "value1": "Cancelling Open Buy Order for {pair}",
             "value2": "limit {limit:8f}",
             "value3": "{stake_amount:8f} {stake_currency}"
         },
-         "webhookentryfill": {
+         "entry_fill": {
             "value1": "Buy Order for {pair} filled",
             "value2": "at {open_rate:8f}",
             "value3": ""
         },
-        "webhookexit": {
+        "exit": {
             "value1": "Exiting {pair}",
             "value2": "limit {limit:8f}",
             "value3": "profit: {profit_amount:8f} {stake_currency} ({profit_ratio})"
         },
-        "webhookexitcancel": {
+        "exit_cancel": {
             "value1": "Cancelling Open Exit Order for {pair}",
             "value2": "limit {limit:8f}",
             "value3": "profit: {profit_amount:8f} {stake_currency} ({profit_ratio})"
         },
-        "webhookexitfill": {
+        "exit_fill": {
             "value1": "Exit Order for {pair} filled",
             "value2": "at {close_rate:8f}.",
             "value3": ""
         },
-        "webhookstatus": {
+        "status": {
             "value1": "Status: {status}",
             "value2": "",
             "value3": ""
@@ -57,7 +57,7 @@ You can set the POST body format to Form-Encoded (default), JSON-Encoded, or raw
         "enabled": true,
         "url": "https://<YOURSUBDOMAIN>.cloud.mattermost.com/hooks/<YOURHOOK>",
         "format": "json",
-        "webhookstatus": {
+        "status": {
             "text": "Status: {status}"
         }
     },
@@ -88,17 +88,30 @@ Optional parameters are available to enable automatic retries for webhook messag
         "url": "https://<YOURHOOKURL>",
         "retries": 3,
         "retry_delay": 0.2,
-        "webhookstatus": {
+        "status": {
             "status": "Status: {status}"
         }
     },
 ```
 
+Custom messages can be sent to Webhook endpoints via the `self.dp.send_msg()` function from within the strategy. To enable this, set the `allow_custom_messages` option to `true`:
+
+```json
+  "webhook": {
+        "enabled": true,
+        "url": "https://<YOURHOOKURL>",
+        "allow_custom_messages": true,
+        "strategy_msg": {
+            "status": "StrategyMessage: {msg}"
+        }
+    },
+```
+
 Different payloads can be configured for different events. Not all fields are necessary, but you should configure at least one of the dicts, otherwise the webhook will never be called.
 
-### Webhookentry
+### Entry
 
-The fields in `webhook.webhookentry` are filled when the bot executes a long/short. Parameters are filled using string.format.
+The fields in `webhook.entry` are filled when the bot executes a long/short. Parameters are filled using string.format.
 Possible parameters are:
 
 * `trade_id`
@@ -118,9 +131,9 @@ Possible parameters are:
 * `current_rate`
 * `enter_tag`
 
-### Webhookentrycancel
+### Entry cancel
 
-The fields in `webhook.webhookentrycancel` are filled when the bot cancels a long/short order. Parameters are filled using string.format.
+The fields in `webhook.entry_cancel` are filled when the bot cancels a long/short order. Parameters are filled using string.format.
 Possible parameters are:
 
 * `trade_id`
@@ -139,9 +152,9 @@ Possible parameters are:
 * `current_rate`
 * `enter_tag`
 
-### Webhookentryfill
+### Entry fill
 
-The fields in `webhook.webhookentryfill` are filled when the bot filled a long/short order. Parameters are filled using string.format.
+The fields in `webhook.entry_fill` are filled when the bot filled a long/short order. Parameters are filled using string.format.
 Possible parameters are:
 
 * `trade_id`
@@ -160,9 +173,9 @@ Possible parameters are:
 * `current_rate`
 * `enter_tag`
 
-### Webhookexit
+### Exit
 
-The fields in `webhook.webhookexit` are filled when the bot exits a trade. Parameters are filled using string.format.
+The fields in `webhook.exit` are filled when the bot exits a trade. Parameters are filled using string.format.
 Possible parameters are:
 
 * `trade_id`
@@ -184,9 +197,9 @@ Possible parameters are:
 * `open_date`
 * `close_date`
 
-### Webhookexitfill
+### Exit fill
 
-The fields in `webhook.webhookexitfill` are filled when the bot fills a exit order (closes a Trade). Parameters are filled using string.format.
+The fields in `webhook.exit_fill` are filled when the bot fills a exit order (closes a Trade). Parameters are filled using string.format.
 Possible parameters are:
 
 * `trade_id`
@@ -209,9 +222,9 @@ Possible parameters are:
 * `open_date`
 * `close_date`
 
-### Webhookexitcancel
+### Exit cancel
 
-The fields in `webhook.webhookexitcancel` are filled when the bot cancels a exit order. Parameters are filled using string.format.
+The fields in `webhook.exit_cancel` are filled when the bot cancels a exit order. Parameters are filled using string.format.
 Possible parameters are:
 
 * `trade_id`
@@ -234,9 +247,9 @@ Possible parameters are:
 * `open_date`
 * `close_date`
 
-### Webhookstatus
+### Status
 
-The fields in `webhook.webhookstatus` are used for regular status messages (Started / Stopped / ...). Parameters are filled using string.format.
+The fields in `webhook.status` are used for regular status messages (Started / Stopped / ...). Parameters are filled using string.format.
 
 The only possible value here is `{status}`.
 
@@ -280,7 +293,6 @@ You can configure this as follows:
 }
 ```
 
-
 The above represents the default (`exit_fill` and `entry_fill` are optional and will default to the above configuration) - modifications are obviously possible.
 
 Available fields correspond to the fields for webhooks and are documented in the corresponding webhook sections.
@@ -288,3 +300,13 @@ Available fields correspond to the fields for webhooks and are documented in the
 The notifications will look as follows by default.
 
 ![discord-notification](assets/discord_notification.png)
+
+Custom messages can be sent from a strategy to Discord endpoints via the dataprovider.send_msg() function. To enable this, set the `allow_custom_messages` option to `true`:
+
+```json
+  "discord": {
+        "enabled": true,
+        "webhook_url": "https://discord.com/api/webhooks/<Your webhook URL ...>",
+        "allow_custom_messages": true,
+    },
+```

docs/windows_installation.md

@@ -3,15 +3,16 @@
 We **strongly** recommend that Windows users use [Docker](docker_quickstart.md) as this will work much easier and smoother (also more secure).
 
 If that is not possible, try using the Windows Linux subsystem (WSL) - for which the Ubuntu instructions should work.
-Otherwise, try the instructions below.
+Otherwise, please follow the instructions below.
 
 ## Install freqtrade manually
 
-!!! Note
-    Make sure to use 64bit Windows and 64bit Python to avoid problems with backtesting or hyperopt due to the memory constraints 32bit applications have under Windows.
+!!! Note "64bit Python version"
+    Please make sure to use 64bit Windows and 64bit Python to avoid problems with backtesting or hyperopt due to the memory constraints 32bit applications have under Windows.
+    32bit python versions are no longer supported under Windows.
 
 !!! Hint
-    Using the [Anaconda Distribution](https://www.anaconda.com/distribution/) under Windows can greatly help with installation problems. Check out the [Anaconda installation section](installation.md#Anaconda) in this document for more information.
+    Using the [Anaconda Distribution](https://www.anaconda.com/distribution/) under Windows can greatly help with installation problems. Check out the [Anaconda installation section](installation.md#installation-with-conda) in the documentation for more information.
 
 ### 1. Clone the git repository
 
@@ -34,7 +35,7 @@ python -m venv .env
 .env\Scripts\activate.ps1
 # optionally install ta-lib from wheel
 # Eventually adjust the below filename to match the downloaded wheel
-pip install --find-links build_helpers\ TA-Lib
+pip install --find-links build_helpers\ TA-Lib -U
 pip install -r requirements.txt
 pip install -e .
 freqtrade

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2022.9'
+__version__ = '2022.11'
 
 if 'dev' in __version__:
     try:
@@ -16,6 +16,6 @@ if 'dev' in __version__:
             from pathlib import Path
             versionfile = Path('./freqtrade_commit')
             if versionfile.is_file():
-                __version__ = f"docker-{versionfile.read_text()[:8]}"
+                __version__ = f"docker-{__version__}-{versionfile.read_text()[:8]}"
         except Exception:
             pass

freqtrade/commands/__init__.py

@@ -15,9 +15,9 @@ from freqtrade.commands.db_commands import start_convert_db
 from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui,
                                                 start_new_strategy)
 from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show
-from freqtrade.commands.list_commands import (start_list_exchanges, start_list_markets,
-                                              start_list_strategies, start_list_timeframes,
-                                              start_show_trades)
+from freqtrade.commands.list_commands import (start_list_exchanges, start_list_freqAI_models,
+                                              start_list_markets, start_list_strategies,
+                                              start_list_timeframes, start_show_trades)
 from freqtrade.commands.optimize_commands import (start_backtesting, start_backtesting_show,
                                                   start_edge, start_hyperopt)
 from freqtrade.commands.pairlist_commands import start_test_pairlist

freqtrade/commands/arguments.py

@@ -25,7 +25,8 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
 ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
                                         "enable_protections", "dry_run_wallet", "timeframe_detail",
                                         "strategy_list", "export", "exportfilename",
-                                        "backtest_breakdown", "backtest_cache"]
+                                        "backtest_breakdown", "backtest_cache",
+                                        "freqai_backtest_live_models"]
 
 ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
                                         "position_stacking", "use_max_market_positions",
@@ -41,6 +42,8 @@ ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]
 ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column", "print_colorized",
                         "recursive_strategy_search"]
 
+ARGS_LIST_FREQAIMODELS = ["freqaimodel_path", "print_one_column", "print_colorized"]
+
 ARGS_LIST_HYPEROPTS = ["hyperopt_path", "print_one_column", "print_colorized"]
 
 ARGS_BACKTEST_SHOW = ["exportfilename", "backtest_show_pair_list"]
@@ -106,8 +109,8 @@ ARGS_ANALYZE_ENTRIES_EXITS = ["exportfilename", "analysis_groups", "enter_reason
                               "exit_reason_list", "indicator_list"]
 
 NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
-                    "list-markets", "list-pairs", "list-strategies", "list-data",
-                    "hyperopt-list", "hyperopt-show", "backtest-filter",
+                    "list-markets", "list-pairs", "list-strategies", "list-freqaimodels",
+                    "list-data", "hyperopt-list", "hyperopt-show", "backtest-filter",
                     "plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv"]
 
 NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]
@@ -192,10 +195,11 @@ class Arguments:
                                         start_create_userdir, start_download_data, start_edge,
                                         start_hyperopt, start_hyperopt_list, start_hyperopt_show,
                                         start_install_ui, start_list_data, start_list_exchanges,
-                                        start_list_markets, start_list_strategies,
-                                        start_list_timeframes, start_new_config, start_new_strategy,
-                                        start_plot_dataframe, start_plot_profit, start_show_trades,
-                                        start_test_pairlist, start_trading, start_webserver)
+                                        start_list_freqAI_models, start_list_markets,
+                                        start_list_strategies, start_list_timeframes,
+                                        start_new_config, start_new_strategy, start_plot_dataframe,
+                                        start_plot_profit, start_show_trades, start_test_pairlist,
+                                        start_trading, start_webserver)
 
         subparsers = self.parser.add_subparsers(dest='command',
                                                 # Use custom message when no subhandler is added
@@ -362,6 +366,15 @@ class Arguments:
         list_strategies_cmd.set_defaults(func=start_list_strategies)
         self._build_args(optionlist=ARGS_LIST_STRATEGIES, parser=list_strategies_cmd)
 
+        # Add list-freqAI Models subcommand
+        list_freqaimodels_cmd = subparsers.add_parser(
+            'list-freqaimodels',
+            help='Print available freqAI models.',
+            parents=[_common_parser],
+        )
+        list_freqaimodels_cmd.set_defaults(func=start_list_freqAI_models)
+        self._build_args(optionlist=ARGS_LIST_FREQAIMODELS, parser=list_freqaimodels_cmd)
+
         # Add list-timeframes subcommand
         list_timeframes_cmd = subparsers.add_parser(
             'list-timeframes',

freqtrade/commands/build_config_commands.py

@@ -108,7 +108,6 @@ def ask_user_config() -> Dict[str, Any]:
                 "binance",
                 "binanceus",
                 "bittrex",
-                "ftx",
                 "gateio",
                 "huobi",
                 "kraken",

freqtrade/commands/cli_options.py

@@ -49,7 +49,7 @@ AVAILABLE_CLI_OPTIONS = {
         default=0,
     ),
     "logfile": Arg(
-        '--logfile',
+        '--logfile', '--log-file',
         help="Log to the file specified. Special values are: 'syslog', 'journald'. "
              "See the documentation for more details.",
         metavar='FILE',
@@ -668,4 +668,9 @@ AVAILABLE_CLI_OPTIONS = {
         help='Specify additional lookup path for freqaimodels.',
         metavar='PATH',
     ),
+    "freqai_backtest_live_models": Arg(
+        '--freqai-backtest-live-models',
+        help='Run backtest with ready models.',
+        action='store_true'
+    ),
 }

freqtrade/commands/list_commands.py

@@ -1,7 +1,6 @@
 import csv
 import logging
 import sys
-from pathlib import Path
 from typing import Any, Dict, List
 
 import rapidjson
@@ -10,7 +9,6 @@ from colorama import init as colorama_init
 from tabulate import tabulate
 
 from freqtrade.configuration import setup_utils_configuration
-from freqtrade.constants import USERPATH_STRATEGIES
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import market_is_active, validate_exchanges
@@ -41,7 +39,7 @@ def start_list_exchanges(args: Dict[str, Any]) -> None:
         print(tabulate(exchanges, headers=['Exchange name', 'Valid', 'reason']))
 
 
-def _print_objs_tabular(objs: List, print_colorized: bool, base_dir: Path) -> None:
+def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
     if print_colorized:
         colorama_init(autoreset=True)
         red = Fore.RED
@@ -55,7 +53,7 @@ def _print_objs_tabular(objs: List, print_colorized: bool, base_dir: Path) -> No
     names = [s['name'] for s in objs]
     objs_to_print = [{
         'name': s['name'] if s['name'] else "--",
-        'location': s['location'].relative_to(base_dir),
+        'location': s['location_rel'],
         'status': (red + "LOAD FAILED" + reset if s['class'] is None
                    else "OK" if names.count(s['name']) == 1
                    else yellow + "DUPLICATE NAME" + reset)
@@ -76,9 +74,8 @@ def start_list_strategies(args: Dict[str, Any]) -> None:
     """
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
-    directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES))
     strategy_objs = StrategyResolver.search_all_objects(
-        directory, not args['print_one_column'], config.get('recursive_strategy_search', False))
+        config, not args['print_one_column'], config.get('recursive_strategy_search', False))
     # Sort alphabetically
     strategy_objs = sorted(strategy_objs, key=lambda x: x['name'])
     for obj in strategy_objs:
@@ -90,7 +87,22 @@ def start_list_strategies(args: Dict[str, Any]) -> None:
     if args['print_one_column']:
         print('\n'.join([s['name'] for s in strategy_objs]))
     else:
-        _print_objs_tabular(strategy_objs, config.get('print_colorized', False), directory)
+        _print_objs_tabular(strategy_objs, config.get('print_colorized', False))
+
+
+def start_list_freqAI_models(args: Dict[str, Any]) -> None:
+    """
+    Print files with FreqAI models custom classes available in the directory
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+    from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
+    model_objs = FreqaiModelResolver.search_all_objects(config, not args['print_one_column'])
+    # Sort alphabetically
+    model_objs = sorted(model_objs, key=lambda x: x['name'])
+    if args['print_one_column']:
+        print('\n'.join([s['name'] for s in model_objs]))
+    else:
+        _print_objs_tabular(model_objs, config.get('print_colorized', False))
 
 
 def start_list_timeframes(args: Dict[str, Any]) -> None:

freqtrade/configuration/__init__.py

@@ -1,6 +1,5 @@
 # flake8: noqa: F401
 
-from freqtrade.configuration.check_exchange import check_exchange
 from freqtrade.configuration.config_setup import setup_utils_configuration
 from freqtrade.configuration.config_validation import validate_config_consistency
 from freqtrade.configuration.configuration import Configuration

freqtrade/configuration/config_validation.py

@@ -86,6 +86,8 @@ def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False)
     _validate_unlimited_amount(conf)
     _validate_ask_orderbook(conf)
     _validate_freqai_hyperopt(conf)
+    _validate_freqai_backtest(conf)
+    _validate_freqai_include_timeframes(conf)
     _validate_consumers(conf)
     validate_migrated_strategy_settings(conf)
 
@@ -334,6 +336,46 @@ def _validate_freqai_hyperopt(conf: Dict[str, Any]) -> None:
             'Using analyze-per-epoch parameter is not supported with a FreqAI strategy.')
 
 
+def _validate_freqai_include_timeframes(conf: Dict[str, Any]) -> None:
+    freqai_enabled = conf.get('freqai', {}).get('enabled', False)
+    if freqai_enabled:
+        main_tf = conf.get('timeframe', '5m')
+        freqai_include_timeframes = conf.get('freqai', {}).get('feature_parameters', {}
+                                                               ).get('include_timeframes', [])
+
+        from freqtrade.exchange import timeframe_to_seconds
+        main_tf_s = timeframe_to_seconds(main_tf)
+        offending_lines = []
+        for tf in freqai_include_timeframes:
+            tf_s = timeframe_to_seconds(tf)
+            if tf_s < main_tf_s:
+                offending_lines.append(tf)
+        if offending_lines:
+            raise OperationalException(
+                f"Main timeframe of {main_tf} must be smaller or equal to FreqAI "
+                f"`include_timeframes`.Offending include-timeframes: {', '.join(offending_lines)}")
+
+
+def _validate_freqai_backtest(conf: Dict[str, Any]) -> None:
+    if conf.get('runmode', RunMode.OTHER) == RunMode.BACKTEST:
+        freqai_enabled = conf.get('freqai', {}).get('enabled', False)
+        timerange = conf.get('timerange')
+        freqai_backtest_live_models = conf.get('freqai_backtest_live_models', False)
+        if freqai_backtest_live_models and freqai_enabled and timerange:
+            raise OperationalException(
+                'Using timerange parameter is not supported with '
+                '--freqai-backtest-live-models parameter.')
+
+        if freqai_backtest_live_models and not freqai_enabled:
+            raise OperationalException(
+                'Using --freqai-backtest-live-models parameter is only '
+                'supported with a FreqAI strategy.')
+
+        if freqai_enabled and not freqai_backtest_live_models and not timerange:
+            raise OperationalException(
+                'Please pass --timerange if you intend to use FreqAI for backtesting.')
+
+
 def _validate_consumers(conf: Dict[str, Any]) -> None:
     emc_conf = conf.get('external_message_consumer', {})
     if emc_conf.get('enabled', False):

freqtrade/configuration/configuration.py

@@ -8,7 +8,6 @@ from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional
 
 from freqtrade import constants
-from freqtrade.configuration.check_exchange import check_exchange
 from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
 from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
 from freqtrade.configuration.environment_vars import enironment_vars_to_dict
@@ -100,6 +99,9 @@ class Configuration:
 
         self._process_freqai_options(config)
 
+        # Import check_exchange here to avoid import cycle problems
+        from freqtrade.exchange.check_exchange import check_exchange
+
         # Check if the exchange set by the user is supported
         check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))
 
@@ -277,6 +279,9 @@ class Configuration:
         self._args_to_config(config, argname='disableparamexport',
                              logstring='Parameter --disableparamexport detected: {} ...')
 
+        self._args_to_config(config, argname='freqai_backtest_live_models',
+                             logstring='Parameter --freqai-backtest-live-models detected ...')
+
         # Edge section:
         if 'stoploss_range' in self.args and self.args["stoploss_range"]:
             txt_range = eval(self.args["stoploss_range"])

freqtrade/configuration/directory_operations.py

@@ -3,7 +3,8 @@ import shutil
 from pathlib import Path
 from typing import Optional
 
-from freqtrade.constants import USER_DATA_FILES, Config
+from freqtrade.constants import (USER_DATA_FILES, USERPATH_FREQAIMODELS, USERPATH_HYPEROPTS,
+                                 USERPATH_NOTEBOOKS, USERPATH_STRATEGIES, Config)
 from freqtrade.exceptions import OperationalException
 
 
@@ -49,8 +50,8 @@ def create_userdata_dir(directory: str, create_dir: bool = False) -> Path:
     :param create_dir: Create directory if it does not exist.
     :return: Path object containing the directory
     """
-    sub_dirs = ["backtest_results", "data", "hyperopts", "hyperopt_results", "logs",
-                "notebooks", "plot", "strategies", ]
+    sub_dirs = ["backtest_results", "data", USERPATH_HYPEROPTS, "hyperopt_results", "logs",
+                USERPATH_NOTEBOOKS, "plot", USERPATH_STRATEGIES, USERPATH_FREQAIMODELS]
     folder = Path(directory)
     chown_user_directory(folder)
     if not folder.is_dir():

freqtrade/configuration/timerange.py

@@ -3,11 +3,12 @@ This module contains the argument manager class
 """
 import logging
 import re
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Optional
 
 import arrow
 
+from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.exceptions import OperationalException
 
 
@@ -29,6 +30,52 @@ class TimeRange:
         self.startts: int = startts
         self.stopts: int = stopts
 
+    @property
+    def startdt(self) -> Optional[datetime]:
+        if self.startts:
+            return datetime.fromtimestamp(self.startts, tz=timezone.utc)
+        return None
+
+    @property
+    def stopdt(self) -> Optional[datetime]:
+        if self.stopts:
+            return datetime.fromtimestamp(self.stopts, tz=timezone.utc)
+        return None
+
+    @property
+    def timerange_str(self) -> str:
+        """
+        Returns a string representation of the timerange as used by parse_timerange.
+        Follows the format yyyymmdd-yyyymmdd - leaving out the parts that are not set.
+        """
+        start = ''
+        stop = ''
+        if startdt := self.startdt:
+            start = startdt.strftime('%Y%m%d')
+        if stopdt := self.stopdt:
+            stop = stopdt.strftime('%Y%m%d')
+        return f"{start}-{stop}"
+
+    @property
+    def start_fmt(self) -> str:
+        """
+        Returns a string representation of the start date
+        """
+        val = 'unbounded'
+        if (startdt := self.startdt) is not None:
+            val = startdt.strftime(DATETIME_PRINT_FORMAT)
+        return val
+
+    @property
+    def stop_fmt(self) -> str:
+        """
+        Returns a string representation of the stop date
+        """
+        val = 'unbounded'
+        if (stopdt := self.stopdt) is not None:
+            val = stopdt.strftime(DATETIME_PRINT_FORMAT)
+        return val
+
     def __eq__(self, other):
         """Override the default Equals behavior"""
         return (self.starttype == other.starttype and self.stoptype == other.stoptype

freqtrade/constants.py

@@ -5,7 +5,7 @@ bot constants
 """
 from typing import Any, Dict, List, Literal, Tuple
 
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, RPCMessageType
 
 
 DEFAULT_CONFIG = 'config.json'
@@ -31,7 +31,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
                          'CalmarHyperOptLoss',
                          'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
                          'ProfitDrawDownHyperOptLoss']
-AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
+AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList',
                        'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
                        'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
                        'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
@@ -159,6 +159,7 @@ CONF_SCHEMA = {
         'ignore_buying_expired_candle_after': {'type': 'number'},
         'trading_mode': {'type': 'string', 'enum': TRADING_MODES},
         'margin_mode': {'type': 'string', 'enum': MARGIN_MODES},
+        'reduce_df_footprint': {'type': 'boolean', 'default': False},
         'liquidation_buffer': {'type': 'number', 'minimum': 0.0, 'maximum': 0.99},
         'backtest_breakdown': {
             'type': 'array',
@@ -282,6 +283,7 @@ CONF_SCHEMA = {
                 'enabled': {'type': 'boolean'},
                 'token': {'type': 'string'},
                 'chat_id': {'type': 'string'},
+                'allow_custom_messages': {'type': 'boolean', 'default': True},
                 'balance_dust_level': {'type': 'number', 'minimum': 0.0},
                 'notification_settings': {
                     'type': 'object',
@@ -344,6 +346,8 @@ CONF_SCHEMA = {
                 'format': {'type': 'string', 'enum': WEBHOOK_FORMAT_OPTIONS, 'default': 'form'},
                 'retries': {'type': 'integer', 'minimum': 0},
                 'retry_delay': {'type': 'number', 'minimum': 0},
+                **dict([(x, {'type': 'object'}) for x in RPCMessageType]),
+                # Below -> Deprecated
                 'webhookentry': {'type': 'object'},
                 'webhookentrycancel': {'type': 'object'},
                 'webhookentryfill': {'type': 'object'},
@@ -508,6 +512,7 @@ CONF_SCHEMA = {
                                 'minimum': 0,
                                 'maximum': 65535
                             },
+                            'secure': {'type': 'boolean', 'default': False},
                             'ws_token': {'type': 'string'},
                         },
                         'required': ['name', 'host', 'ws_token']
@@ -537,7 +542,9 @@ CONF_SCHEMA = {
             "properties": {
                 "enabled": {"type": "boolean", "default": False},
                 "keras": {"type": "boolean", "default": False},
-                "conv_width": {"type": "integer", "default": 2},
+                "write_metrics_to_disk": {"type": "boolean", "default": False},
+                "purge_old_models": {"type": "boolean", "default": True},
+                "conv_width": {"type": "integer", "default": 1},
                 "train_period_days": {"type": "integer", "default": 0},
                 "backtest_period_days": {"type": "number", "default": 7},
                 "identifier": {"type": "string", "default": "example"},
@@ -567,6 +574,7 @@ CONF_SCHEMA = {
                     "properties": {
                         "test_size": {"type": "number"},
                         "random_state": {"type": "integer"},
+                        "shuffle": {"type": "boolean", "default": False}
                     },
                 },
                 "model_training_parameters": {
@@ -652,5 +660,6 @@ LongShort = Literal['long', 'short']
 EntryExit = Literal['entry', 'exit']
 BuySell = Literal['buy', 'sell']
 MakerTaker = Literal['maker', 'taker']
+BidAsk = Literal['bid', 'ask']
 
 Config = Dict[str, Any]

freqtrade/data/btanalysis.py

@@ -26,7 +26,7 @@ BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
                    'profit_ratio', 'profit_abs', 'exit_reason',
                    'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
                    'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'enter_tag',
-                   'is_short', 'open_timestamp', 'close_timestamp', 'orders'
+                   'leverage', 'is_short', 'open_timestamp', 'close_timestamp', 'orders'
                    ]
 
 
@@ -280,6 +280,8 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
             # Compatibility support for pre short Columns
             if 'is_short' not in df.columns:
                 df['is_short'] = 0
+            if 'leverage' not in df.columns:
+                df['leverage'] = 1.0
             if 'enter_tag' not in df.columns:
                 df['enter_tag'] = df['buy_tag']
                 df = df.drop(['buy_tag'], axis=1)

freqtrade/data/converter.py

@@ -3,10 +3,10 @@ Functions to convert data from one format to another
 """
 import itertools
 import logging
-from datetime import datetime, timezone
 from operator import itemgetter
 from typing import Dict, List
 
+import numpy as np
 import pandas as pd
 from pandas import DataFrame, to_datetime
 
@@ -47,8 +47,7 @@ def ohlcv_to_dataframe(ohlcv: list, timeframe: str, pair: str, *,
 
 
 def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *,
-                          fill_missing: bool = True,
-                          drop_incomplete: bool = True) -> DataFrame:
+                          fill_missing: bool, drop_incomplete: bool) -> DataFrame:
     """
     Cleanse a OHLCV dataframe by
       * Grouping it by date (removes duplicate tics)
@@ -138,11 +137,9 @@ def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date',
         df = df.iloc[startup_candles:, :]
     else:
         if timerange.starttype == 'date':
-            start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
-            df = df.loc[df[df_date_col] >= start, :]
+            df = df.loc[df[df_date_col] >= timerange.startdt, :]
     if timerange.stoptype == 'date':
-        stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
-        df = df.loc[df[df_date_col] <= stop, :]
+        df = df.loc[df[df_date_col] <= timerange.stopdt, :]
     return df
 
 
@@ -314,3 +311,29 @@ def convert_ohlcv_format(
                 if erase and convert_from != convert_to:
                     logger.info(f"Deleting source data for {pair} / {timeframe}")
                     src.ohlcv_purge(pair=pair, timeframe=timeframe, candle_type=candle_type)
+
+
+def reduce_dataframe_footprint(df: DataFrame) -> DataFrame:
+    """
+    Ensure all values are float32 in the incoming dataframe.
+    :param df: Dataframe to be converted to float/int 32s
+    :return: Dataframe converted to float/int 32s
+    """
+
+    logger.debug(f"Memory usage of dataframe is "
+                 f"{df.memory_usage().sum() / 1024**2:.2f} MB")
+
+    df_dtypes = df.dtypes
+    for column, dtype in df_dtypes.items():
+        if column in ['open', 'high', 'low', 'close', 'volume']:
+            continue
+        if dtype == np.float64:
+            df_dtypes[column] = np.float32
+        elif dtype == np.int64:
+            df_dtypes[column] = np.int32
+    df = df.astype(df_dtypes)
+
+    logger.debug(f"Memory usage after optimization is: "
+                 f"{df.memory_usage().sum() / 1024**2:.2f} MB")
+
+    return df

freqtrade/data/history/history_utils.py

@@ -1,6 +1,6 @@
 import logging
 import operator
-from datetime import datetime, timezone
+from datetime import datetime
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple
 
@@ -26,7 +26,7 @@ def load_pair_history(pair: str,
                       datadir: Path, *,
                       timerange: Optional[TimeRange] = None,
                       fill_up_missing: bool = True,
-                      drop_incomplete: bool = True,
+                      drop_incomplete: bool = False,
                       startup_candles: int = 0,
                       data_format: str = None,
                       data_handler: IDataHandler = None,
@@ -160,9 +160,9 @@ def _load_cached_data_for_updating(
     end = None
     if timerange:
         if timerange.starttype == 'date':
-            start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
+            start = timerange.startdt
         if timerange.stoptype == 'date':
-            end = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
+            end = timerange.stopdt
 
     # Intentionally don't pass timerange in - since we need to load the full dataset.
     data = data_handler.ohlcv_load(pair, timeframe=timeframe,

freqtrade/data/history/idatahandler.py

@@ -102,6 +102,11 @@ class IDataHandler(ABC):
         :return: (min, max)
         """
         data = self._ohlcv_load(pair, timeframe, None, candle_type)
+        if data.empty:
+            return (
+                datetime.fromtimestamp(0, tz=timezone.utc),
+                datetime.fromtimestamp(0, tz=timezone.utc)
+            )
         return data.iloc[0]['date'].to_pydatetime(), data.iloc[-1]['date'].to_pydatetime()
 
     @abstractmethod
@@ -275,7 +280,7 @@ class IDataHandler(ABC):
                    candle_type: CandleType, *,
                    timerange: Optional[TimeRange] = None,
                    fill_missing: bool = True,
-                   drop_incomplete: bool = True,
+                   drop_incomplete: bool = False,
                    startup_candles: int = 0,
                    warn_no_data: bool = True,
                    ) -> DataFrame:
@@ -303,7 +308,7 @@ class IDataHandler(ABC):
             timerange=timerange_startup,
             candle_type=candle_type
         )
-        if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data):
+        if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data, True):
             return pairdf
         else:
             enddate = pairdf.iloc[-1]['date']
@@ -323,8 +328,9 @@ class IDataHandler(ABC):
             self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data)
             return pairdf
 
-    def _check_empty_df(self, pairdf: DataFrame, pair: str, timeframe: str,
-                        candle_type: CandleType, warn_no_data: bool):
+    def _check_empty_df(
+            self, pairdf: DataFrame, pair: str, timeframe: str, candle_type: CandleType,
+            warn_no_data: bool, warn_price: bool = False) -> bool:
         """
         Warn on empty dataframe
         """
@@ -335,6 +341,20 @@ class IDataHandler(ABC):
                     "Use `freqtrade download-data` to download the data"
                 )
             return True
+        elif warn_price:
+            candle_price_gap = 0
+            if (candle_type in (CandleType.SPOT, CandleType.FUTURES) and
+                    not pairdf.empty
+                    and 'close' in pairdf.columns and 'open' in pairdf.columns):
+                # Detect gaps between prior close and open
+                gaps = ((pairdf['open'] - pairdf['close'].shift(1)) / pairdf['close'].shift(1))
+                gaps = gaps.dropna()
+                if len(gaps):
+                    candle_price_gap = max(abs(gaps))
+            if candle_price_gap > 0.1:
+                logger.info(f"Price jump in {pair}, {timeframe}, {candle_type} between two candles "
+                            f"of {candle_price_gap:.2%} detected.")
+
         return False
 
     def _validate_pairdata(self, pair, pairdata: DataFrame, timeframe: str,
@@ -346,13 +366,11 @@ class IDataHandler(ABC):
         """
 
         if timerange.starttype == 'date':
-            start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
-            if pairdata.iloc[0]['date'] > start:
+            if pairdata.iloc[0]['date'] > timerange.startdt:
                 logger.warning(f"{pair}, {candle_type}, {timeframe}, "
                                f"data starts at {pairdata.iloc[0]['date']:%Y-%m-%d %H:%M:%S}")
         if timerange.stoptype == 'date':
-            stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
-            if pairdata.iloc[-1]['date'] < stop:
+            if pairdata.iloc[-1]['date'] < timerange.stopdt:
                 logger.warning(f"{pair}, {candle_type}, {timeframe}, "
                                f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}")
 

freqtrade/edge/edge_positioning.py

@@ -392,7 +392,7 @@ class Edge:
         # Returning a list of pairs in order of "expectancy"
         return final
 
-    def _find_trades_for_stoploss_range(self, df, pair, stoploss_range):
+    def _find_trades_for_stoploss_range(self, df, pair: str, stoploss_range) -> list:
         buy_column = df['enter_long'].values
         sell_column = df['exit_long'].values
         date_column = df['date'].values
@@ -407,7 +407,7 @@ class Edge:
         return result
 
     def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column,
-                                        ohlc_columns, stoploss, pair):
+                                        ohlc_columns, stoploss, pair: str):
         """
         Iterate through ohlc_columns in order to find the next trade
         Next trade opens from the first buy signal noticed to

freqtrade/exchange/__init__.py

@@ -9,15 +9,15 @@ from freqtrade.exchange.bitpanda import Bitpanda
 from freqtrade.exchange.bittrex import Bittrex
 from freqtrade.exchange.bybit import Bybit
 from freqtrade.exchange.coinbasepro import Coinbasepro
-from freqtrade.exchange.exchange import (amount_to_contract_precision, amount_to_contracts,
-                                         amount_to_precision, available_exchanges, ccxt_exchanges,
-                                         contracts_to_amount, date_minus_candles,
-                                         is_exchange_known_ccxt, is_exchange_officially_supported,
-                                         market_is_active, price_to_precision, timeframe_to_minutes,
-                                         timeframe_to_msecs, timeframe_to_next_date,
-                                         timeframe_to_prev_date, timeframe_to_seconds,
-                                         validate_exchange, validate_exchanges)
-from freqtrade.exchange.ftx import Ftx
+from freqtrade.exchange.exchange_utils import (amount_to_contract_precision, amount_to_contracts,
+                                               amount_to_precision, available_exchanges,
+                                               ccxt_exchanges, contracts_to_amount,
+                                               date_minus_candles, is_exchange_known_ccxt,
+                                               market_is_active, price_to_precision,
+                                               timeframe_to_minutes, timeframe_to_msecs,
+                                               timeframe_to_next_date, timeframe_to_prev_date,
+                                               timeframe_to_seconds, validate_exchange,
+                                               validate_exchanges)
 from freqtrade.exchange.gateio import Gateio
 from freqtrade.exchange.hitbtc import Hitbtc
 from freqtrade.exchange.huobi import Huobi

freqtrade/exchange/binance.py

@@ -11,6 +11,7 @@ from freqtrade.enums import CandleType, MarginMode, TradingMode
 from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError
 from freqtrade.exchange import Exchange
 from freqtrade.exchange.common import retrier
+from freqtrade.exchange.types import Tickers
 from freqtrade.misc import deep_merge_dicts, json_load
 
 
@@ -41,25 +42,7 @@ class Binance(Exchange):
         (TradingMode.FUTURES, MarginMode.ISOLATED)
     ]
 
-    def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool:
-        """
-        Verify stop_loss against stoploss-order value (limit or price)
-        Returns True if adjustment is necessary.
-        :param side: "buy" or "sell"
-        """
-        order_types = ('stop_loss_limit', 'stop', 'stop_market')
-
-        return (
-            order.get('stopPrice', None) is None
-            or (
-                order['type'] in order_types
-                and (
-                    (side == "sell" and stop_loss > float(order['stopPrice'])) or
-                    (side == "buy" and stop_loss < float(order['stopPrice']))
-                )
-            ))
-
-    def get_tickers(self, symbols: Optional[List[str]] = None, cached: bool = False) -> Dict:
+    def get_tickers(self, symbols: Optional[List[str]] = None, cached: bool = False) -> Tickers:
         tickers = super().get_tickers(symbols=symbols, cached=cached)
         if self.trading_mode == TradingMode.FUTURES:
             # Binance's future result has no bid/ask values.
@@ -68,6 +51,37 @@ class Binance(Exchange):
             tickers = deep_merge_dicts(bidsasks, tickers, allow_null_overrides=False)
         return tickers
 
+    @retrier
+    def additional_exchange_init(self) -> None:
+        """
+        Additional exchange initialization logic.
+        .api will be available at this point.
+        Must be overridden in child methods if required.
+        """
+        try:
+            if self.trading_mode == TradingMode.FUTURES and not self._config['dry_run']:
+                position_side = self._api.fapiPrivateGetPositionsideDual()
+                self._log_exchange_response('position_side_setting', position_side)
+                assets_margin = self._api.fapiPrivateGetMultiAssetsMargin()
+                self._log_exchange_response('multi_asset_margin', assets_margin)
+                msg = ""
+                if position_side.get('dualSidePosition') is True:
+                    msg += (
+                        "\nHedge Mode is not supported by freqtrade. "
+                        "Please change 'Position Mode' on your binance futures account.")
+                if assets_margin.get('multiAssetsMargin') is True:
+                    msg += ("\nMulti-Asset Mode is not supported by freqtrade. "
+                            "Please change 'Asset Mode' on your binance futures account.")
+                if msg:
+                    raise OperationalException(msg)
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(
+                f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e
+        except ccxt.BaseError as e:
+            raise OperationalException(e) from e
+
     @retrier
     def _set_leverage(
         self,

freqtrade/exchange/bybit.py

@@ -20,8 +20,12 @@ class Bybit(Exchange):
     """
 
     _ft_has: Dict = {
-        "ohlcv_candle_limit": 200,
-        "ccxt_futures_name": "linear"
+        "ohlcv_candle_limit": 1000,
+        "ccxt_futures_name": "linear",
+        "ohlcv_has_history": False,
+    }
+    _ft_has_futures: Dict = {
+        "ohlcv_has_history": True,
     }
 
     _supported_trading_mode_margin_pairs: List[Tuple[TradingMode, MarginMode]] = [

freqtrade/exchange/check_exchange.py

@@ -3,8 +3,8 @@ import logging
 from freqtrade.constants import Config
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
-from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt,
-                                is_exchange_officially_supported, validate_exchange)
+from freqtrade.exchange import available_exchanges, is_exchange_known_ccxt, validate_exchange
+from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS, SUPPORTED_EXCHANGES
 
 
 logger = logging.getLogger(__name__)
@@ -52,7 +52,7 @@ def check_exchange(config: Config, check_for_bad: bool = True) -> bool:
         else:
             logger.warning(f'Exchange "{exchange}"  will not work with Freqtrade. Reason: {reason}')
 
-    if is_exchange_officially_supported(exchange):
+    if MAP_EXCHANGE_CHILDCLASS.get(exchange, exchange) in SUPPORTED_EXCHANGES:
         logger.info(f'Exchange "{exchange}" is officially supported '
                     f'by the Freqtrade development team.')
     else:

freqtrade/exchange/common.py

@@ -52,7 +52,6 @@ MAP_EXCHANGE_CHILDCLASS = {
 SUPPORTED_EXCHANGES = [
     'binance',
     'bittrex',
-    'ftx',
     'gateio',
     'huobi',
     'kraken',

freqtrade/exchange/exchange.py

@@ -8,7 +8,6 @@ import inspect
 import logging
 from copy import deepcopy
 from datetime import datetime, timedelta, timezone
-from math import ceil
 from threading import Lock
 from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union
 
@@ -16,29 +15,31 @@ import arrow
 import ccxt
 import ccxt.async_support as ccxt_async
 from cachetools import TTLCache
-from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, decimal_to_precision
+from ccxt import TICK_SIZE
 from dateutil import parser
-from pandas import DataFrame
+from pandas import DataFrame, concat
 
-from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell,
-                                 Config, EntryExit, ListPairsWithTimeframes, MakerTaker,
+from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BidAsk,
+                                 BuySell, Config, EntryExit, ListPairsWithTimeframes, MakerTaker,
                                  PairWithTimeframe)
-from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
+from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_dict_to_list
 from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, TradingMode
 from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
                                   InvalidOrderException, OperationalException, PricingError,
                                   RetryableOrderError, TemporaryError)
-from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES,
-                                       EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED,
-                                       SUPPORTED_EXCHANGES, remove_credentials, retrier,
+from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, remove_credentials, retrier,
                                        retrier_async)
+from freqtrade.exchange.exchange_utils import (CcxtModuleType, amount_to_contract_precision,
+                                               amount_to_contracts, amount_to_precision,
+                                               contracts_to_amount, date_minus_candles,
+                                               is_exchange_known_ccxt, market_is_active,
+                                               price_to_precision, timeframe_to_minutes,
+                                               timeframe_to_msecs, timeframe_to_next_date,
+                                               timeframe_to_prev_date, timeframe_to_seconds)
+from freqtrade.exchange.types import Ticker, Tickers
 from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_json,
                             safe_value_fallback2)
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
-from freqtrade.util import FtPrecise
-
-
-CcxtModuleType = Any
 
 
 logger = logging.getLogger(__name__)
@@ -180,13 +181,14 @@ class Exchange:
             exchange_config, ccxt_async, ccxt_kwargs=ccxt_async_config)
 
         logger.info(f'Using Exchange "{self.name}"')
-
+        self.required_candle_call_count = 1
         if validate:
             # Initial markets load
             self._load_markets()
             self.validate_config(config)
+            self._startup_candle_count: int = config.get('startup_candle_count', 0)
             self.required_candle_call_count = self.validate_required_startup_candles(
-                config.get('startup_candle_count', 0), config.get('timeframe', ''))
+                self._startup_candle_count, config.get('timeframe', ''))
 
         # Converts the interval provided in minutes in config to seconds
         self.markets_refresh_interval: int = exchange_config.get(
@@ -409,11 +411,13 @@ class Exchange:
         else:
             return DataFrame()
 
-    def get_contract_size(self, pair: str) -> float:
+    def get_contract_size(self, pair: str) -> Optional[float]:
         if self.trading_mode == TradingMode.FUTURES:
-            market = self.markets[pair]
+            market = self.markets.get(pair, {})
             contract_size: float = 1.0
-            if market['contractSize'] is not None:
+            if not market:
+                return None
+            if market.get('contractSize') is not None:
                 # ccxt has contractSize in markets as string
                 contract_size = float(market['contractSize'])
             return contract_size
@@ -1073,7 +1077,14 @@ class Exchange:
         Verify stop_loss against stoploss-order value (limit or price)
         Returns True if adjustment is necessary.
         """
-        raise OperationalException(f"stoploss is not implemented for {self.name}.")
+        if not self._ft_has.get('stoploss_on_exchange'):
+            raise OperationalException(f"stoploss is not implemented for {self.name}.")
+
+        return (
+            order.get('stopPrice', None) is None
+            or ((side == "sell" and stop_loss > float(order['stopPrice'])) or
+                (side == "buy" and stop_loss < float(order['stopPrice'])))
+        )
 
     def _get_stop_order_type(self, user_order_type) -> Tuple[str, str]:
 
@@ -1103,7 +1114,7 @@ class Exchange:
                 'In stoploss limit order, stop price should be more than limit price')
         return limit_rate
 
-    def _get_stop_params(self, ordertype: str, stop_price: float) -> Dict:
+    def _get_stop_params(self, side: BuySell, ordertype: str, stop_price: float) -> Dict:
         params = self._params.copy()
         # Verify if stopPrice works for your exchange!
         params.update({'stopPrice': stop_price})
@@ -1152,7 +1163,8 @@ class Exchange:
             return dry_order
 
         try:
-            params = self._get_stop_params(ordertype=ordertype, stop_price=stop_price_norm)
+            params = self._get_stop_params(side=side, ordertype=ordertype,
+                                           stop_price=stop_price_norm)
             if self.trading_mode == TradingMode.FUTURES:
                 params['reduceOnly'] = True
 
@@ -1292,7 +1304,14 @@ class Exchange:
             order = self.fetch_order(order_id, pair)
         except InvalidOrderException:
             logger.warning(f"Could not fetch cancelled order {order_id}.")
-            order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
+            order = {
+                'id': order_id,
+                'status': 'canceled',
+                'amount': amount,
+                'filled': 0.0,
+                'fee': {},
+                'info': {}
+            }
 
         return order
 
@@ -1413,14 +1432,17 @@ class Exchange:
             raise OperationalException(e) from e
 
     @retrier
-    def get_tickers(self, symbols: Optional[List[str]] = None, cached: bool = False) -> Dict:
+    def get_tickers(self, symbols: Optional[List[str]] = None, cached: bool = False) -> Tickers:
         """
         :param cached: Allow cached result
         :return: fetch_tickers result
         """
+        tickers: Tickers
+        if not self.exchange_has('fetchTickers'):
+            return {}
         if cached:
             with self._cache_lock:
-                tickers = self._fetch_tickers_cache.get('fetch_tickers')
+                tickers = self._fetch_tickers_cache.get('fetch_tickers')  # type: ignore
             if tickers:
                 return tickers
         try:
@@ -1443,12 +1465,12 @@ class Exchange:
     # Pricing info
 
     @retrier
-    def fetch_ticker(self, pair: str) -> dict:
+    def fetch_ticker(self, pair: str) -> Ticker:
         try:
             if (pair not in self.markets or
                     self.markets[pair].get('active', False) is False):
                 raise ExchangeError(f"Pair {pair} not available")
-            data = self._api.fetch_ticker(pair)
+            data: Ticker = self._api.fetch_ticker(pair)
             return data
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
@@ -1499,7 +1521,7 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
-    def _get_price_side(self, side: str, is_short: bool, conf_strategy: Dict) -> str:
+    def _get_price_side(self, side: str, is_short: bool, conf_strategy: Dict) -> BidAsk:
         price_side = conf_strategy['price_side']
 
         if price_side in ('same', 'other'):
@@ -1518,7 +1540,7 @@ class Exchange:
 
     def get_rate(self, pair: str, refresh: bool,
                  side: EntryExit, is_short: bool,
-                 order_book: Optional[dict] = None, ticker: Optional[dict] = None) -> float:
+                 order_book: Optional[dict] = None, ticker: Optional[Ticker] = None) -> float:
         """
         Calculates bid/ask target
         bid rate - between current ask price and last price
@@ -1667,6 +1689,17 @@ class Exchange:
     @retrier
     def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1,
                 price: float = 1, taker_or_maker: MakerTaker = 'maker') -> float:
+        """
+        Retrieve fee from exchange
+        :param symbol: Pair
+        :param type: Type of order (market, limit, ...)
+        :param side: Side of order (buy, sell)
+        :param amount: Amount of order
+        :param price: Price of order
+        :param taker_or_maker: 'maker' or 'taker' (ignored if "type" is provided)
+        """
+        if type and type == 'market':
+            taker_or_maker = 'taker'
         try:
             if self._config['dry_run'] and self._config.get('fee', None) is not None:
                 return self._config['fee']
@@ -1844,10 +1877,22 @@ class Exchange:
         return pair, timeframe, candle_type, data
 
     def _build_coroutine(self, pair: str, timeframe: str, candle_type: CandleType,
-                         since_ms: Optional[int]) -> Coroutine:
+                         since_ms: Optional[int], cache: bool) -> Coroutine:
+        not_all_data = cache and self.required_candle_call_count > 1
+        if cache and (pair, timeframe, candle_type) in self._klines:
+            candle_limit = self.ohlcv_candle_limit(timeframe, candle_type)
+            min_date = date_minus_candles(timeframe, candle_limit - 5).timestamp()
+            # Check if 1 call can get us updated candles without hole in the data.
+            if min_date < self._pairs_last_refresh_time.get((pair, timeframe, candle_type), 0):
+                # Cache can be used - do one-off call.
+                not_all_data = False
+            else:
+                # Time jump detected, evict cache
+                logger.info(
+                    f"Time jump detected. Evicting cache for {pair}, {timeframe}, {candle_type}")
+                del self._klines[(pair, timeframe, candle_type)]
 
-        if (not since_ms
-                and (self._ft_has["ohlcv_require_since"] or self.required_candle_call_count > 1)):
+        if (not since_ms and (self._ft_has["ohlcv_require_since"] or not_all_data)):
             # Multiple calls for one pair - to get more history
             one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(
                 timeframe, candle_type, since_ms)
@@ -1863,6 +1908,60 @@ class Exchange:
             return self._async_get_candle_history(
                 pair, timeframe, since_ms=since_ms, candle_type=candle_type)
 
+    def _build_ohlcv_dl_jobs(
+            self, pair_list: ListPairsWithTimeframes, since_ms: Optional[int],
+            cache: bool) -> Tuple[List[Coroutine], List[Tuple[str, str, CandleType]]]:
+        """
+        Build Coroutines to execute as part of refresh_latest_ohlcv
+        """
+        input_coroutines = []
+        cached_pairs = []
+        for pair, timeframe, candle_type in set(pair_list):
+            if (timeframe not in self.timeframes
+                    and candle_type in (CandleType.SPOT, CandleType.FUTURES)):
+                logger.warning(
+                    f"Cannot download ({pair}, {timeframe}) combination as this timeframe is "
+                    f"not available on {self.name}. Available timeframes are "
+                    f"{', '.join(self.timeframes)}.")
+                continue
+
+            if ((pair, timeframe, candle_type) not in self._klines or not cache
+                    or self._now_is_time_to_refresh(pair, timeframe, candle_type)):
+
+                input_coroutines.append(
+                    self._build_coroutine(pair, timeframe, candle_type, since_ms, cache))
+
+            else:
+                logger.debug(
+                    f"Using cached candle (OHLCV) data for {pair}, {timeframe}, {candle_type} ..."
+                )
+                cached_pairs.append((pair, timeframe, candle_type))
+
+        return input_coroutines, cached_pairs
+
+    def _process_ohlcv_df(self, pair: str, timeframe: str, c_type: CandleType, ticks: List[List],
+                          cache: bool, drop_incomplete: bool) -> DataFrame:
+        # keeping last candle time as last refreshed time of the pair
+        if ticks and cache:
+            self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[-1][0] // 1000
+        # keeping parsed dataframe in cache
+        ohlcv_df = ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True,
+                                      drop_incomplete=drop_incomplete)
+        if cache:
+            if (pair, timeframe, c_type) in self._klines:
+                old = self._klines[(pair, timeframe, c_type)]
+                # Reassign so we return the updated, combined df
+                ohlcv_df = clean_ohlcv_dataframe(concat([old, ohlcv_df], axis=0), timeframe, pair,
+                                                 fill_missing=True, drop_incomplete=False)
+                candle_limit = self.ohlcv_candle_limit(timeframe, self._config['candle_type_def'])
+                # Age out old candles
+                ohlcv_df = ohlcv_df.tail(candle_limit + self._startup_candle_count)
+                ohlcv_df = ohlcv_df.reset_index(drop=True)
+                self._klines[(pair, timeframe, c_type)] = ohlcv_df
+            else:
+                self._klines[(pair, timeframe, c_type)] = ohlcv_df
+        return ohlcv_df
+
     def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *,
                              since_ms: Optional[int] = None, cache: bool = True,
                              drop_incomplete: Optional[bool] = None
@@ -1880,27 +1979,9 @@ class Exchange:
         """
         logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list))
         drop_incomplete = self._ohlcv_partial_candle if drop_incomplete is None else drop_incomplete
-        input_coroutines = []
-        cached_pairs = []
-        # Gather coroutines to run
-        for pair, timeframe, candle_type in set(pair_list):
-            if (timeframe not in self.timeframes
-                    and candle_type in (CandleType.SPOT, CandleType.FUTURES)):
-                logger.warning(
-                    f"Cannot download ({pair}, {timeframe}) combination as this timeframe is "
-                    f"not available on {self.name}. Available timeframes are "
-                    f"{', '.join(self.timeframes)}.")
-                continue
-            if ((pair, timeframe, candle_type) not in self._klines or not cache
-                    or self._now_is_time_to_refresh(pair, timeframe, candle_type)):
-                input_coroutines.append(self._build_coroutine(
-                    pair, timeframe, candle_type=candle_type, since_ms=since_ms))
 
-            else:
-                logger.debug(
-                    f"Using cached candle (OHLCV) data for {pair}, {timeframe}, {candle_type} ..."
-                )
-                cached_pairs.append((pair, timeframe, candle_type))
+        # Gather coroutines to run
+        input_coroutines, cached_pairs = self._build_ohlcv_dl_jobs(pair_list, since_ms, cache)
 
         results_df = {}
         # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling
@@ -1917,16 +1998,11 @@ class Exchange:
                     continue
                 # Deconstruct tuple (has 4 elements)
                 pair, timeframe, c_type, ticks = res
-                # keeping last candle time as last refreshed time of the pair
-                if ticks:
-                    self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[-1][0] // 1000
-                # keeping parsed dataframe in cache
-                ohlcv_df = ohlcv_to_dataframe(
-                    ticks, timeframe, pair=pair, fill_missing=True,
-                    drop_incomplete=drop_incomplete)
+                ohlcv_df = self._process_ohlcv_df(
+                    pair, timeframe, c_type, ticks, cache, drop_incomplete)
+
                 results_df[(pair, timeframe, c_type)] = ohlcv_df
-                if cache:
-                    self._klines[(pair, timeframe, c_type)] = ohlcv_df
+
         # Return cached klines
         for pair, timeframe, c_type in cached_pairs:
             results_df[(pair, timeframe, c_type)] = self.klines(
@@ -1939,13 +2015,8 @@ class Exchange:
     def _now_is_time_to_refresh(self, pair: str, timeframe: str, candle_type: CandleType) -> bool:
         # Timeframe in seconds
         interval_in_sec = timeframe_to_seconds(timeframe)
-
-        return not (
-            (self._pairs_last_refresh_time.get(
-                (pair, timeframe, candle_type),
-                0
-            ) + interval_in_sec) >= arrow.utcnow().int_timestamp
-        )
+        plr = self._pairs_last_refresh_time.get((pair, timeframe, candle_type), 0) + interval_in_sec
+        return plr < arrow.utcnow().int_timestamp
 
     @retrier_async
     async def _async_get_candle_history(
@@ -1971,8 +2042,8 @@ class Exchange:
             candle_limit = self.ohlcv_candle_limit(
                 timeframe, candle_type=candle_type, since_ms=since_ms)
 
-            if candle_type != CandleType.SPOT:
-                params.update({'price': candle_type})
+            if candle_type and candle_type != CandleType.SPOT:
+                params.update({'price': candle_type.value})
             if candle_type != CandleType.FUNDING_RATE:
                 data = await self._api_async.fetch_ohlcv(
                     pair, timeframe=timeframe, since=since_ms,
@@ -2748,244 +2819,3 @@ class Exchange:
             # describes the min amt for a tier, and the lowest tier will always go down to 0
         else:
             raise OperationalException(f"Cannot get maintenance ratio using {self.name}")
-
-
-def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = None) -> bool:
-    return exchange_name in ccxt_exchanges(ccxt_module)
-
-
-def is_exchange_officially_supported(exchange_name: str) -> bool:
-    return exchange_name in SUPPORTED_EXCHANGES
-
-
-def ccxt_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]:
-    """
-    Return the list of all exchanges known to ccxt
-    """
-    return ccxt_module.exchanges if ccxt_module is not None else ccxt.exchanges
-
-
-def available_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]:
-    """
-    Return exchanges available to the bot, i.e. non-bad exchanges in the ccxt list
-    """
-    exchanges = ccxt_exchanges(ccxt_module)
-    return [x for x in exchanges if validate_exchange(x)[0]]
-
-
-def validate_exchange(exchange: str) -> Tuple[bool, str]:
-    ex_mod = getattr(ccxt, exchange.lower())()
-    if not ex_mod or not ex_mod.has:
-        return False, ''
-    missing = [k for k in EXCHANGE_HAS_REQUIRED if ex_mod.has.get(k) is not True]
-    if missing:
-        return False, f"missing: {', '.join(missing)}"
-
-    missing_opt = [k for k in EXCHANGE_HAS_OPTIONAL if not ex_mod.has.get(k)]
-
-    if exchange.lower() in BAD_EXCHANGES:
-        return False, BAD_EXCHANGES.get(exchange.lower(), '')
-    if missing_opt:
-        return True, f"missing opt: {', '.join(missing_opt)}"
-
-    return True, ''
-
-
-def validate_exchanges(all_exchanges: bool) -> List[Tuple[str, bool, str]]:
-    """
-    :return: List of tuples with exchangename, valid, reason.
-    """
-    exchanges = ccxt_exchanges() if all_exchanges else available_exchanges()
-    exchanges_valid = [
-        (e, *validate_exchange(e)) for e in exchanges
-    ]
-    return exchanges_valid
-
-
-def timeframe_to_seconds(timeframe: str) -> int:
-    """
-    Translates the timeframe interval value written in the human readable
-    form ('1m', '5m', '1h', '1d', '1w', etc.) to the number
-    of seconds for one timeframe interval.
-    """
-    return ccxt.Exchange.parse_timeframe(timeframe)
-
-
-def timeframe_to_minutes(timeframe: str) -> int:
-    """
-    Same as timeframe_to_seconds, but returns minutes.
-    """
-    return ccxt.Exchange.parse_timeframe(timeframe) // 60
-
-
-def timeframe_to_msecs(timeframe: str) -> int:
-    """
-    Same as timeframe_to_seconds, but returns milliseconds.
-    """
-    return ccxt.Exchange.parse_timeframe(timeframe) * 1000
-
-
-def timeframe_to_prev_date(timeframe: str, date: datetime = None) -> datetime:
-    """
-    Use Timeframe and determine the candle start date for this date.
-    Does not round when given a candle start date.
-    :param timeframe: timeframe in string format (e.g. "5m")
-    :param date: date to use. Defaults to now(utc)
-    :returns: date of previous candle (with utc timezone)
-    """
-    if not date:
-        date = datetime.now(timezone.utc)
-
-    new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000,
-                                                  ROUND_DOWN) // 1000
-    return datetime.fromtimestamp(new_timestamp, tz=timezone.utc)
-
-
-def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime:
-    """
-    Use Timeframe and determine next candle.
-    :param timeframe: timeframe in string format (e.g. "5m")
-    :param date: date to use. Defaults to now(utc)
-    :returns: date of next candle (with utc timezone)
-    """
-    if not date:
-        date = datetime.now(timezone.utc)
-    new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000,
-                                                  ROUND_UP) // 1000
-    return datetime.fromtimestamp(new_timestamp, tz=timezone.utc)
-
-
-def date_minus_candles(
-        timeframe: str, candle_count: int, date: Optional[datetime] = None) -> datetime:
-    """
-    subtract X candles from a date.
-    :param timeframe: timeframe in string format (e.g. "5m")
-    :param candle_count: Amount of candles to subtract.
-    :param date: date to use. Defaults to now(utc)
-
-    """
-    if not date:
-        date = datetime.now(timezone.utc)
-
-    tf_min = timeframe_to_minutes(timeframe)
-    new_date = timeframe_to_prev_date(timeframe, date) - timedelta(minutes=tf_min * candle_count)
-    return new_date
-
-
-def market_is_active(market: Dict) -> bool:
-    """
-    Return True if the market is active.
-    """
-    # "It's active, if the active flag isn't explicitly set to false. If it's missing or
-    # true then it's true. If it's undefined, then it's most likely true, but not 100% )"
-    # See https://github.com/ccxt/ccxt/issues/4874,
-    # https://github.com/ccxt/ccxt/issues/4075#issuecomment-434760520
-    return market.get('active', True) is not False
-
-
-def amount_to_contracts(amount: float, contract_size: Optional[float]) -> float:
-    """
-    Convert amount to contracts.
-    :param amount: amount to convert
-    :param contract_size: contract size - taken from exchange.get_contract_size(pair)
-    :return: num-contracts
-    """
-    if contract_size and contract_size != 1:
-        return float(FtPrecise(amount) / FtPrecise(contract_size))
-    else:
-        return amount
-
-
-def contracts_to_amount(num_contracts: float, contract_size: Optional[float]) -> float:
-    """
-    Takes num-contracts and converts it to contract size
-    :param num_contracts: number of contracts
-    :param contract_size: contract size - taken from exchange.get_contract_size(pair)
-    :return: Amount
-    """
-
-    if contract_size and contract_size != 1:
-        return float(FtPrecise(num_contracts) * FtPrecise(contract_size))
-    else:
-        return num_contracts
-
-
-def amount_to_precision(amount: float, amount_precision: Optional[float],
-                        precisionMode: Optional[int]) -> float:
-    """
-    Returns the amount to buy or sell to a precision the Exchange accepts
-    Re-implementation of ccxt internal methods - ensuring we can test the result is correct
-    based on our definitions.
-    :param amount: amount to truncate
-    :param amount_precision: amount precision to use.
-                             should be retrieved from markets[pair]['precision']['amount']
-    :param precisionMode: precision mode to use. Should be used from precisionMode
-                          one of ccxt's DECIMAL_PLACES, SIGNIFICANT_DIGITS, or TICK_SIZE
-    :return: truncated amount
-    """
-    if amount_precision is not None and precisionMode is not None:
-        precision = int(amount_precision) if precisionMode != TICK_SIZE else amount_precision
-        # precision must be an int for non-ticksize inputs.
-        amount = float(decimal_to_precision(amount, rounding_mode=TRUNCATE,
-                                            precision=precision,
-                                            counting_mode=precisionMode,
-                                            ))
-
-    return amount
-
-
-def amount_to_contract_precision(
-        amount, amount_precision: Optional[float], precisionMode: Optional[int],
-        contract_size: Optional[float]) -> float:
-    """
-    Returns the amount to buy or sell to a precision the Exchange accepts
-    including calculation to and from contracts.
-    Re-implementation of ccxt internal methods - ensuring we can test the result is correct
-    based on our definitions.
-    :param amount: amount to truncate
-    :param amount_precision: amount precision to use.
-                             should be retrieved from markets[pair]['precision']['amount']
-    :param precisionMode: precision mode to use. Should be used from precisionMode
-                          one of ccxt's DECIMAL_PLACES, SIGNIFICANT_DIGITS, or TICK_SIZE
-    :param contract_size: contract size - taken from exchange.get_contract_size(pair)
-    :return: truncated amount
-    """
-    if amount_precision is not None and precisionMode is not None:
-        contracts = amount_to_contracts(amount, contract_size)
-        amount_p = amount_to_precision(contracts, amount_precision, precisionMode)
-        return contracts_to_amount(amount_p, contract_size)
-    return amount
-
-
-def price_to_precision(price: float, price_precision: Optional[float],
-                       precisionMode: Optional[int]) -> float:
-    """
-    Returns the price rounded up to the precision the Exchange accepts.
-    Partial Re-implementation of ccxt internal method decimal_to_precision(),
-    which does not support rounding up
-    TODO: If ccxt supports ROUND_UP for decimal_to_precision(), we could remove this and
-    align with amount_to_precision().
-    !!! Rounds up
-    :param price: price to convert
-    :param price_precision: price precision to use. Used from markets[pair]['precision']['price']
-    :param precisionMode: precision mode to use. Should be used from precisionMode
-                          one of ccxt's DECIMAL_PLACES, SIGNIFICANT_DIGITS, or TICK_SIZE
-    :return: price rounded up to the precision the Exchange accepts
-
-    """
-    if price_precision is not None and precisionMode is not None:
-        # price = float(decimal_to_precision(price, rounding_mode=ROUND,
-        #                                    precision=price_precision,
-        #                                    counting_mode=self.precisionMode,
-        #                                    ))
-        if precisionMode == TICK_SIZE:
-            precision = FtPrecise(price_precision)
-            price_str = FtPrecise(price)
-            missing = price_str % precision
-            if not missing == FtPrecise("0"):
-                price = round(float(str(price_str - missing + precision)), 14)
-        else:
-            symbol_prec = price_precision
-            big_price = price * pow(10, symbol_prec)
-            price = ceil(big_price) / pow(10, symbol_prec)
-    return price

freqtrade/exchange/exchange_utils.py

@@ -0,0 +1,252 @@
+"""
+Exchange support utils
+"""
+from datetime import datetime, timedelta, timezone
+from math import ceil
+from typing import Any, Dict, List, Optional, Tuple
+
+import ccxt
+from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, decimal_to_precision
+
+from freqtrade.exchange.common import BAD_EXCHANGES, EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED
+from freqtrade.util import FtPrecise
+
+
+CcxtModuleType = Any
+
+
+def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = None) -> bool:
+    return exchange_name in ccxt_exchanges(ccxt_module)
+
+
+def ccxt_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]:
+    """
+    Return the list of all exchanges known to ccxt
+    """
+    return ccxt_module.exchanges if ccxt_module is not None else ccxt.exchanges
+
+
+def available_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]:
+    """
+    Return exchanges available to the bot, i.e. non-bad exchanges in the ccxt list
+    """
+    exchanges = ccxt_exchanges(ccxt_module)
+    return [x for x in exchanges if validate_exchange(x)[0]]
+
+
+def validate_exchange(exchange: str) -> Tuple[bool, str]:
+    ex_mod = getattr(ccxt, exchange.lower())()
+    if not ex_mod or not ex_mod.has:
+        return False, ''
+    missing = [k for k in EXCHANGE_HAS_REQUIRED if ex_mod.has.get(k) is not True]
+    if missing:
+        return False, f"missing: {', '.join(missing)}"
+
+    missing_opt = [k for k in EXCHANGE_HAS_OPTIONAL if not ex_mod.has.get(k)]
+
+    if exchange.lower() in BAD_EXCHANGES:
+        return False, BAD_EXCHANGES.get(exchange.lower(), '')
+    if missing_opt:
+        return True, f"missing opt: {', '.join(missing_opt)}"
+
+    return True, ''
+
+
+def validate_exchanges(all_exchanges: bool) -> List[Tuple[str, bool, str]]:
+    """
+    :return: List of tuples with exchangename, valid, reason.
+    """
+    exchanges = ccxt_exchanges() if all_exchanges else available_exchanges()
+    exchanges_valid = [
+        (e, *validate_exchange(e)) for e in exchanges
+    ]
+    return exchanges_valid
+
+
+def timeframe_to_seconds(timeframe: str) -> int:
+    """
+    Translates the timeframe interval value written in the human readable
+    form ('1m', '5m', '1h', '1d', '1w', etc.) to the number
+    of seconds for one timeframe interval.
+    """
+    return ccxt.Exchange.parse_timeframe(timeframe)
+
+
+def timeframe_to_minutes(timeframe: str) -> int:
+    """
+    Same as timeframe_to_seconds, but returns minutes.
+    """
+    return ccxt.Exchange.parse_timeframe(timeframe) // 60
+
+
+def timeframe_to_msecs(timeframe: str) -> int:
+    """
+    Same as timeframe_to_seconds, but returns milliseconds.
+    """
+    return ccxt.Exchange.parse_timeframe(timeframe) * 1000
+
+
+def timeframe_to_prev_date(timeframe: str, date: datetime = None) -> datetime:
+    """
+    Use Timeframe and determine the candle start date for this date.
+    Does not round when given a candle start date.
+    :param timeframe: timeframe in string format (e.g. "5m")
+    :param date: date to use. Defaults to now(utc)
+    :returns: date of previous candle (with utc timezone)
+    """
+    if not date:
+        date = datetime.now(timezone.utc)
+
+    new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000,
+                                                  ROUND_DOWN) // 1000
+    return datetime.fromtimestamp(new_timestamp, tz=timezone.utc)
+
+
+def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime:
+    """
+    Use Timeframe and determine next candle.
+    :param timeframe: timeframe in string format (e.g. "5m")
+    :param date: date to use. Defaults to now(utc)
+    :returns: date of next candle (with utc timezone)
+    """
+    if not date:
+        date = datetime.now(timezone.utc)
+    new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000,
+                                                  ROUND_UP) // 1000
+    return datetime.fromtimestamp(new_timestamp, tz=timezone.utc)
+
+
+def date_minus_candles(
+        timeframe: str, candle_count: int, date: Optional[datetime] = None) -> datetime:
+    """
+    subtract X candles from a date.
+    :param timeframe: timeframe in string format (e.g. "5m")
+    :param candle_count: Amount of candles to subtract.
+    :param date: date to use. Defaults to now(utc)
+
+    """
+    if not date:
+        date = datetime.now(timezone.utc)
+
+    tf_min = timeframe_to_minutes(timeframe)
+    new_date = timeframe_to_prev_date(timeframe, date) - timedelta(minutes=tf_min * candle_count)
+    return new_date
+
+
+def market_is_active(market: Dict) -> bool:
+    """
+    Return True if the market is active.
+    """
+    # "It's active, if the active flag isn't explicitly set to false. If it's missing or
+    # true then it's true. If it's undefined, then it's most likely true, but not 100% )"
+    # See https://github.com/ccxt/ccxt/issues/4874,
+    # https://github.com/ccxt/ccxt/issues/4075#issuecomment-434760520
+    return market.get('active', True) is not False
+
+
+def amount_to_contracts(amount: float, contract_size: Optional[float]) -> float:
+    """
+    Convert amount to contracts.
+    :param amount: amount to convert
+    :param contract_size: contract size - taken from exchange.get_contract_size(pair)
+    :return: num-contracts
+    """
+    if contract_size and contract_size != 1:
+        return float(FtPrecise(amount) / FtPrecise(contract_size))
+    else:
+        return amount
+
+
+def contracts_to_amount(num_contracts: float, contract_size: Optional[float]) -> float:
+    """
+    Takes num-contracts and converts it to contract size
+    :param num_contracts: number of contracts
+    :param contract_size: contract size - taken from exchange.get_contract_size(pair)
+    :return: Amount
+    """
+
+    if contract_size and contract_size != 1:
+        return float(FtPrecise(num_contracts) * FtPrecise(contract_size))
+    else:
+        return num_contracts
+
+
+def amount_to_precision(amount: float, amount_precision: Optional[float],
+                        precisionMode: Optional[int]) -> float:
+    """
+    Returns the amount to buy or sell to a precision the Exchange accepts
+    Re-implementation of ccxt internal methods - ensuring we can test the result is correct
+    based on our definitions.
+    :param amount: amount to truncate
+    :param amount_precision: amount precision to use.
+                             should be retrieved from markets[pair]['precision']['amount']
+    :param precisionMode: precision mode to use. Should be used from precisionMode
+                          one of ccxt's DECIMAL_PLACES, SIGNIFICANT_DIGITS, or TICK_SIZE
+    :return: truncated amount
+    """
+    if amount_precision is not None and precisionMode is not None:
+        precision = int(amount_precision) if precisionMode != TICK_SIZE else amount_precision
+        # precision must be an int for non-ticksize inputs.
+        amount = float(decimal_to_precision(amount, rounding_mode=TRUNCATE,
+                                            precision=precision,
+                                            counting_mode=precisionMode,
+                                            ))
+
+    return amount
+
+
+def amount_to_contract_precision(
+        amount, amount_precision: Optional[float], precisionMode: Optional[int],
+        contract_size: Optional[float]) -> float:
+    """
+    Returns the amount to buy or sell to a precision the Exchange accepts
+    including calculation to and from contracts.
+    Re-implementation of ccxt internal methods - ensuring we can test the result is correct
+    based on our definitions.
+    :param amount: amount to truncate
+    :param amount_precision: amount precision to use.
+                             should be retrieved from markets[pair]['precision']['amount']
+    :param precisionMode: precision mode to use. Should be used from precisionMode
+                          one of ccxt's DECIMAL_PLACES, SIGNIFICANT_DIGITS, or TICK_SIZE
+    :param contract_size: contract size - taken from exchange.get_contract_size(pair)
+    :return: truncated amount
+    """
+    if amount_precision is not None and precisionMode is not None:
+        contracts = amount_to_contracts(amount, contract_size)
+        amount_p = amount_to_precision(contracts, amount_precision, precisionMode)
+        return contracts_to_amount(amount_p, contract_size)
+    return amount
+
+
+def price_to_precision(price: float, price_precision: Optional[float],
+                       precisionMode: Optional[int]) -> float:
+    """
+    Returns the price rounded up to the precision the Exchange accepts.
+    Partial Re-implementation of ccxt internal method decimal_to_precision(),
+    which does not support rounding up
+    TODO: If ccxt supports ROUND_UP for decimal_to_precision(), we could remove this and
+    align with amount_to_precision().
+    !!! Rounds up
+    :param price: price to convert
+    :param price_precision: price precision to use. Used from markets[pair]['precision']['price']
+    :param precisionMode: precision mode to use. Should be used from precisionMode
+                          one of ccxt's DECIMAL_PLACES, SIGNIFICANT_DIGITS, or TICK_SIZE
+    :return: price rounded up to the precision the Exchange accepts
+
+    """
+    if price_precision is not None and precisionMode is not None:
+        # price = float(decimal_to_precision(price, rounding_mode=ROUND,
+        #                                    precision=price_precision,
+        #                                    counting_mode=self.precisionMode,
+        #                                    ))
+        if precisionMode == TICK_SIZE:
+            precision = FtPrecise(price_precision)
+            price_str = FtPrecise(price)
+            missing = price_str % precision
+            if not missing == FtPrecise("0"):
+                price = round(float(str(price_str - missing + precision)), 14)
+        else:
+            symbol_prec = price_precision
+            big_price = price * pow(10, symbol_prec)
+            price = ceil(big_price) / pow(10, symbol_prec)
+    return price

freqtrade/exchange/ftx.py

@@ -1,178 +0,0 @@
-""" FTX exchange subclass """
-import logging
-from typing import Any, Dict, List, Optional, Tuple
-
-import ccxt
-
-from freqtrade.constants import BuySell
-from freqtrade.enums import MarginMode, TradingMode
-from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException,
-                                  OperationalException, TemporaryError)
-from freqtrade.exchange import Exchange
-from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT, retrier
-from freqtrade.misc import safe_value_fallback2
-
-
-logger = logging.getLogger(__name__)
-
-
-class Ftx(Exchange):
-
-    _ft_has: Dict = {
-        "order_time_in_force": ['GTC', 'IOC', 'PO'],
-        "stoploss_on_exchange": True,
-        "ohlcv_candle_limit": 1500,
-        "ohlcv_require_since": True,
-        "ohlcv_volume_currency": "quote",
-        "mark_ohlcv_price": "index",
-        "mark_ohlcv_timeframe": "1h",
-    }
-
-    _supported_trading_mode_margin_pairs: List[Tuple[TradingMode, MarginMode]] = [
-        # TradingMode.SPOT always supported and not required in this list
-        # (TradingMode.MARGIN, MarginMode.CROSS),
-        # (TradingMode.FUTURES, MarginMode.CROSS)
-    ]
-
-    def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool:
-        """
-        Verify stop_loss against stoploss-order value (limit or price)
-        Returns True if adjustment is necessary.
-        """
-        return order['type'] == 'stop' and (
-            side == "sell" and stop_loss > float(order['price']) or
-            side == "buy" and stop_loss < float(order['price'])
-        )
-
-    @retrier(retries=0)
-    def stoploss(self, pair: str, amount: float, stop_price: float,
-                 order_types: Dict, side: BuySell, leverage: float) -> Dict:
-        """
-        Creates a stoploss order.
-        depending on order_types.stoploss configuration, uses 'market' or limit order.
-
-        Limit orders are defined by having orderPrice set, otherwise a market order is used.
-        """
-        limit_price_pct = order_types.get('stoploss_on_exchange_limit_ratio', 0.99)
-        if side == "sell":
-            limit_rate = stop_price * limit_price_pct
-        else:
-            limit_rate = stop_price * (2 - limit_price_pct)
-
-        ordertype = "stop"
-
-        stop_price = self.price_to_precision(pair, stop_price)
-
-        if self._config['dry_run']:
-            dry_order = self.create_dry_run_order(
-                pair, ordertype, side, amount, stop_price, leverage, stop_loss=True)
-            return dry_order
-
-        try:
-            params = self._params.copy()
-            if order_types.get('stoploss', 'market') == 'limit':
-                # set orderPrice to place limit order, otherwise it's a market order
-                params['orderPrice'] = limit_rate
-            if self.trading_mode == TradingMode.FUTURES:
-                params.update({'reduceOnly': True})
-
-            params['stopPrice'] = stop_price
-            amount = self.amount_to_precision(pair, amount)
-
-            self._lev_prep(pair, leverage, side)
-            order = self._api.create_order(symbol=pair, type=ordertype, side=side,
-                                           amount=amount, params=params)
-            self._log_exchange_response('create_stoploss_order', order)
-            logger.info('stoploss order added for %s. '
-                        'stop price: %s.', pair, stop_price)
-            return order
-        except ccxt.InsufficientFunds as e:
-            raise InsufficientFundsError(
-                f'Insufficient funds to create {ordertype} {side} order on market {pair}. '
-                f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
-                f'Message: {e}') from e
-        except ccxt.InvalidOrder as e:
-            raise InvalidOrderException(
-                f'Could not create {ordertype} {side} order on market {pair}. '
-                f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
-                f'Message: {e}') from e
-        except ccxt.DDoSProtection as e:
-            raise DDosProtection(e) from e
-        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
-            raise TemporaryError(
-                f'Could not place {side} order due to {e.__class__.__name__}. Message: {e}') from e
-        except ccxt.BaseError as e:
-            raise OperationalException(e) from e
-
-    @retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
-    def fetch_stoploss_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict:
-        if self._config['dry_run']:
-            return self.fetch_dry_run_order(order_id)
-
-        try:
-            orders = self._api.fetch_orders(pair, None, params={'type': 'stop'})
-
-            order = [order for order in orders if order['id'] == order_id]
-            self._log_exchange_response('fetch_stoploss_order', order)
-            if len(order) == 1:
-                if order[0].get('status') == 'closed':
-                    # Trigger order was triggered ...
-                    real_order_id: Optional[str] = order[0].get('info', {}).get('orderId')
-                    # OrderId may be None for stoploss-market orders
-                    # So we need to get it through the endpoint
-                    # /conditional_orders/{conditional_order_id}/triggers
-                    if not real_order_id:
-                        res = self._api.privateGetConditionalOrdersConditionalOrderIdTriggers(
-                            params={'conditional_order_id': order_id})
-                        self._log_exchange_response('fetch_stoploss_order2', res)
-                        real_order_id = res['result'][0]['orderId'] if res.get(
-                            'result', []) else None
-
-                    if real_order_id:
-                        order1 = self._api.fetch_order(real_order_id, pair)
-                        self._log_exchange_response('fetch_stoploss_order1', order1)
-                        # Fake type to stop - as this was really a stop order.
-                        order1['id_stop'] = order1['id']
-                        order1['id'] = order_id
-                        order1['type'] = 'stop'
-                        order1['status_stop'] = 'triggered'
-                        return order1
-
-                return order[0]
-            else:
-                raise InvalidOrderException(f"Could not get stoploss order for id {order_id}")
-
-        except ccxt.InvalidOrder as e:
-            raise InvalidOrderException(
-                f'Tried to get an invalid order (id: {order_id}). Message: {e}') from e
-        except ccxt.DDoSProtection as e:
-            raise DDosProtection(e) from e
-        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
-            raise TemporaryError(
-                f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e
-        except ccxt.BaseError as e:
-            raise OperationalException(e) from e
-
-    @retrier
-    def cancel_stoploss_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict:
-        if self._config['dry_run']:
-            return {}
-        try:
-            order = self._api.cancel_order(order_id, pair, params={'type': 'stop'})
-            self._log_exchange_response('cancel_stoploss_order', order)
-            return order
-        except ccxt.InvalidOrder as e:
-            raise InvalidOrderException(
-                f'Could not cancel order. Message: {e}') from e
-        except ccxt.DDoSProtection as e:
-            raise DDosProtection(e) from e
-        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
-            raise TemporaryError(
-                f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e
-        except ccxt.BaseError as e:
-            raise OperationalException(e) from e
-
-    def get_order_id_conditional(self, order: Dict[str, Any]) -> str:
-        if order['type'] == 'stop':
-            return safe_value_fallback2(order, order, 'id_stop', 'id')
-        return order['id']

freqtrade/exchange/gateio.py

@@ -126,13 +126,3 @@ class Gateio(Exchange):
             pair=pair,
             params={'stop': True}
         )
-
-    def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool:
-        """
-        Verify stop_loss against stoploss-order value (limit or price)
-        Returns True if adjustment is necessary.
-        """
-        return (order.get('stopPrice', None) is None or (
-            side == "sell" and stop_loss > float(order['stopPrice'])) or
-            (side == "buy" and stop_loss < float(order['stopPrice']))
-            )

freqtrade/exchange/huobi.py

@@ -2,6 +2,7 @@
 import logging
 from typing import Dict
 
+from freqtrade.constants import BuySell
 from freqtrade.exchange import Exchange
 
 
@@ -22,20 +23,7 @@ class Huobi(Exchange):
         "l2_limit_range_required": False,
     }
 
-    def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool:
-        """
-        Verify stop_loss against stoploss-order value (limit or price)
-        Returns True if adjustment is necessary.
-        """
-        return (
-            order.get('stopPrice', None) is None
-            or (
-                order['type'] == 'stop'
-                and stop_loss > float(order['stopPrice'])
-            )
-        )
-
-    def _get_stop_params(self, ordertype: str, stop_price: float) -> Dict:
+    def _get_stop_params(self, side: BuySell, ordertype: str, stop_price: float) -> Dict:
 
         params = self._params.copy()
         params.update({

freqtrade/exchange/kraken.py

@@ -12,6 +12,7 @@ from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, Invali
                                   OperationalException, TemporaryError)
 from freqtrade.exchange import Exchange
 from freqtrade.exchange.common import retrier
+from freqtrade.exchange.types import Tickers
 
 
 logger = logging.getLogger(__name__)
@@ -45,7 +46,7 @@ class Kraken(Exchange):
         return (parent_check and
                 market.get('darkpool', False) is False)
 
-    def get_tickers(self, symbols: Optional[List[str]] = None, cached: bool = False) -> Dict:
+    def get_tickers(self, symbols: Optional[List[str]] = None, cached: bool = False) -> Tickers:
         # Only fetch tickers for current stake currency
         # Otherwise the request for kraken becomes too large.
         symbols = list(self.get_markets(quote_currencies=[self._config['stake_currency']]))
@@ -217,3 +218,19 @@ class Kraken(Exchange):
             fees = sum(df['open_fund'] * df['open_mark'] * amount * time_in_ratio)
 
         return fees if is_short else -fees
+
+    def _trades_contracts_to_amount(self, trades: List) -> List:
+        """
+        Fix "last" id issue for kraken data downloads
+        This whole override can probably be removed once the following
+        issue is closed in ccxt: https://github.com/ccxt/ccxt/issues/15827
+        """
+        super()._trades_contracts_to_amount(trades)
+        if (
+            len(trades) > 0
+            and isinstance(trades[-1].get('info'), list)
+            and len(trades[-1].get('info', [])) > 7
+        ):
+
+            trades[-1]['id'] = trades[-1].get('info', [])[-1]
+        return trades

freqtrade/exchange/kucoin.py

@@ -2,6 +2,7 @@
 import logging
 from typing import Dict
 
+from freqtrade.constants import BuySell
 from freqtrade.exchange import Exchange
 
 
@@ -27,17 +28,7 @@ class Kucoin(Exchange):
         "ohlcv_candle_limit": 1500,
     }
 
-    def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool:
-        """
-        Verify stop_loss against stoploss-order value (limit or price)
-        Returns True if adjustment is necessary.
-        """
-        return (
-            order.get('stopPrice', None) is None
-            or stop_loss > float(order['stopPrice'])
-        )
-
-    def _get_stop_params(self, ordertype: str, stop_price: float) -> Dict:
+    def _get_stop_params(self, side: BuySell, ordertype: str, stop_price: float) -> Dict:
 
         params = self._params.copy()
         params.update({

freqtrade/exchange/okx.py

@@ -78,7 +78,8 @@ class Okx(Exchange):
             raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
-                f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e
+                f'Error in additional_exchange_init due to {e.__class__.__name__}. Message: {e}'
+                ) from e
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 

freqtrade/exchange/types.py

@@ -0,0 +1,16 @@
+from typing import Dict, Optional, TypedDict
+
+
+class Ticker(TypedDict):
+    symbol: str
+    ask: Optional[float]
+    askVolume: Optional[float]
+    bid: Optional[float]
+    bidVolume: Optional[float]
+    last: Optional[float]
+    quoteVolume: Optional[float]
+    baseVolume: Optional[float]
+    # Several more - only listing required.
+
+
+Tickers = Dict[str, Ticker]

freqtrade/freqai/base_models/BaseClassifierModel.py

@@ -51,7 +51,7 @@ class BaseClassifierModel(IFreqaiModel):
                     f"{end_date} --------------------")
         # split data into train/test data.
         data_dictionary = dk.make_train_test_datasets(features_filtered, labels_filtered)
-        if not self.freqai_info.get("fit_live_predictions", 0) or not self.live:
+        if not self.freqai_info.get("fit_live_predictions_candles", 0) or not self.live:
             dk.fit_labels()
         # normalize all data based on train_dataset only
         data_dictionary = dk.normalize_data(data_dictionary)
@@ -78,7 +78,7 @@ class BaseClassifierModel(IFreqaiModel):
     ) -> Tuple[DataFrame, npt.NDArray[np.int_]]:
         """
         Filter the prediction features data and predict with it.
-        :param: unfiltered_df: Full dataframe for the current backtest period.
+        :param unfiltered_df: Full dataframe for the current backtest period.
         :return:
         :pred_df: dataframe containing the predictions
         :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove
@@ -92,7 +92,7 @@ class BaseClassifierModel(IFreqaiModel):
         filtered_df = dk.normalize_data_from_metadata(filtered_df)
         dk.data_dictionary["prediction_features"] = filtered_df
 
-        self.data_cleaning_predict(dk, filtered_df)
+        self.data_cleaning_predict(dk)
 
         predictions = self.model.predict(dk.data_dictionary["prediction_features"])
         pred_df = DataFrame(predictions, columns=dk.label_list)

freqtrade/freqai/base_models/BaseRegressionModel.py

@@ -50,7 +50,7 @@ class BaseRegressionModel(IFreqaiModel):
                     f"{end_date} --------------------")
         # split data into train/test data.
         data_dictionary = dk.make_train_test_datasets(features_filtered, labels_filtered)
-        if not self.freqai_info.get("fit_live_predictions", 0) or not self.live:
+        if not self.freqai_info.get("fit_live_predictions_candles", 0) or not self.live:
             dk.fit_labels()
         # normalize all data based on train_dataset only
         data_dictionary = dk.normalize_data(data_dictionary)
@@ -77,7 +77,7 @@ class BaseRegressionModel(IFreqaiModel):
     ) -> Tuple[DataFrame, npt.NDArray[np.int_]]:
         """
         Filter the prediction features data and predict with it.
-        :param: unfiltered_df: Full dataframe for the current backtest period.
+        :param unfiltered_df: Full dataframe for the current backtest period.
         :return:
         :pred_df: dataframe containing the predictions
         :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove
@@ -92,7 +92,7 @@ class BaseRegressionModel(IFreqaiModel):
         dk.data_dictionary["prediction_features"] = filtered_df
 
         # optional additional data cleaning/analysis
-        self.data_cleaning_predict(dk, filtered_df)
+        self.data_cleaning_predict(dk)
 
         predictions = self.model.predict(dk.data_dictionary["prediction_features"])
         pred_df = DataFrame(predictions, columns=dk.label_list)

freqtrade/freqai/base_models/BaseTensorFlowModel.py

@@ -47,7 +47,7 @@ class BaseTensorFlowModel(IFreqaiModel):
                     f"{end_date} --------------------")
         # split data into train/test data.
         data_dictionary = dk.make_train_test_datasets(features_filtered, labels_filtered)
-        if not self.freqai_info.get("fit_live_predictions", 0) or not self.live:
+        if not self.freqai_info.get("fit_live_predictions_candles", 0) or not self.live:
             dk.fit_labels()
         # normalize all data based on train_dataset only
         data_dictionary = dk.normalize_data(data_dictionary)

freqtrade/freqai/base_models/FreqaiMultiOutputClassifier.py

@@ -0,0 +1,93 @@
+import numpy as np
+from joblib import Parallel
+from sklearn.base import is_classifier
+from sklearn.multioutput import MultiOutputClassifier, _fit_estimator
+from sklearn.utils.fixes import delayed
+from sklearn.utils.multiclass import check_classification_targets
+from sklearn.utils.validation import has_fit_parameter
+
+from freqtrade.exceptions import OperationalException
+
+
+class FreqaiMultiOutputClassifier(MultiOutputClassifier):
+
+    def fit(self, X, y, sample_weight=None, fit_params=None):
+        """Fit the model to data, separately for each output variable.
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The input data.
+        y : {array-like, sparse matrix} of shape (n_samples, n_outputs)
+            Multi-output targets. An indicator matrix turns on multilabel
+            estimation.
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights. If `None`, then samples are equally weighted.
+            Only supported if the underlying classifier supports sample
+            weights.
+        fit_params : A list of dicts for the fit_params
+            Parameters passed to the ``estimator.fit`` method of each step.
+            Each dict may contain same or different values (e.g. different
+            eval_sets or init_models)
+            .. versionadded:: 0.23
+        Returns
+        -------
+        self : object
+            Returns a fitted instance.
+        """
+
+        if not hasattr(self.estimator, "fit"):
+            raise ValueError("The base estimator should implement a fit method")
+
+        y = self._validate_data(X="no_validation", y=y, multi_output=True)
+
+        if is_classifier(self):
+            check_classification_targets(y)
+
+        if y.ndim == 1:
+            raise ValueError(
+                "y must have at least two dimensions for "
+                "multi-output regression but has only one."
+            )
+
+        if sample_weight is not None and not has_fit_parameter(
+            self.estimator, "sample_weight"
+        ):
+            raise ValueError("Underlying estimator does not support sample weights.")
+
+        if not fit_params:
+            fit_params = [None] * y.shape[1]
+
+        self.estimators_ = Parallel(n_jobs=self.n_jobs)(
+            delayed(_fit_estimator)(
+                self.estimator, X, y[:, i], sample_weight, **fit_params[i]
+            )
+            for i in range(y.shape[1])
+        )
+
+        self.classes_ = []
+        for estimator in self.estimators_:
+            self.classes_.extend(estimator.classes_)
+        if len(set(self.classes_)) != len(self.classes_):
+            raise OperationalException(f"Class labels must be unique across targets: "
+                                       f"{self.classes_}")
+
+        if hasattr(self.estimators_[0], "n_features_in_"):
+            self.n_features_in_ = self.estimators_[0].n_features_in_
+        if hasattr(self.estimators_[0], "feature_names_in_"):
+            self.feature_names_in_ = self.estimators_[0].feature_names_in_
+
+        return self
+
+    def predict_proba(self, X):
+        """
+        Get predict_proba and stack arrays horizontally
+        """
+        results = np.hstack(super().predict_proba(X))
+        return np.squeeze(results)
+
+    def predict(self, X):
+        """
+        Get predict and squeeze into 2D array
+        """
+        results = super().predict(X)
+        return np.squeeze(results)

freqtrade/freqai/data_drawer.py

@@ -1,14 +1,15 @@
 import collections
-import json
 import logging
 import re
 import shutil
 import threading
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Dict, Tuple, TypedDict
 
 import numpy as np
 import pandas as pd
+import psutil
 import rapidjson
 from joblib import dump, load
 from joblib.externals import cloudpickle
@@ -65,6 +66,8 @@ class FreqaiDataDrawer:
         self.pair_dict: Dict[str, pair_info] = {}
         # dictionary holding all actively inferenced models in memory given a model filename
         self.model_dictionary: Dict[str, Any] = {}
+        # all additional metadata that we want to keep in ram
+        self.meta_data_dictionary: Dict[str, Dict[str, Any]] = {}
         self.model_return_values: Dict[str, DataFrame] = {}
         self.historic_data: Dict[str, Dict[str, DataFrame]] = {}
         self.historic_predictions: Dict[str, DataFrame] = {}
@@ -78,30 +81,60 @@ class FreqaiDataDrawer:
         self.historic_predictions_bkp_path = Path(
             self.full_path / "historic_predictions.backup.pkl")
         self.pair_dictionary_path = Path(self.full_path / "pair_dictionary.json")
+        self.metric_tracker_path = Path(self.full_path / "metric_tracker.json")
         self.follow_mode = follow_mode
         if follow_mode:
             self.create_follower_dict()
         self.load_drawer_from_disk()
         self.load_historic_predictions_from_disk()
+        self.metric_tracker: Dict[str, Dict[str, Dict[str, list]]] = {}
+        self.load_metric_tracker_from_disk()
         self.training_queue: Dict[str, int] = {}
         self.history_lock = threading.Lock()
         self.save_lock = threading.Lock()
         self.pair_dict_lock = threading.Lock()
+        self.metric_tracker_lock = threading.Lock()
         self.old_DBSCAN_eps: Dict[str, float] = {}
         self.empty_pair_dict: pair_info = {
                 "model_filename": "", "trained_timestamp": 0,
                 "data_path": "", "extras": {}}
 
+    def update_metric_tracker(self, metric: str, value: float, pair: str) -> None:
+        """
+        General utility for adding and updating custom metrics. Typically used
+        for adding training performance, train timings, inferenc timings, cpu loads etc.
+        """
+        with self.metric_tracker_lock:
+            if pair not in self.metric_tracker:
+                self.metric_tracker[pair] = {}
+            if metric not in self.metric_tracker[pair]:
+                self.metric_tracker[pair][metric] = {'timestamp': [], 'value': []}
+
+            timestamp = int(datetime.now(timezone.utc).timestamp())
+            self.metric_tracker[pair][metric]['value'].append(value)
+            self.metric_tracker[pair][metric]['timestamp'].append(timestamp)
+
+    def collect_metrics(self, time_spent: float, pair: str):
+        """
+        Add metrics to the metric tracker dictionary
+        """
+        load1, load5, load15 = psutil.getloadavg()
+        cpus = psutil.cpu_count()
+        self.update_metric_tracker('train_time', time_spent, pair)
+        self.update_metric_tracker('cpu_load1min', load1 / cpus, pair)
+        self.update_metric_tracker('cpu_load5min', load5 / cpus, pair)
+        self.update_metric_tracker('cpu_load15min', load15 / cpus, pair)
+
     def load_drawer_from_disk(self):
         """
         Locate and load a previously saved data drawer full of all pair model metadata in
         present model folder.
-        :return: bool - whether or not the drawer was located
+        Load any existing metric tracker that may be present.
         """
         exists = self.pair_dictionary_path.is_file()
         if exists:
             with open(self.pair_dictionary_path, "r") as fp:
-                self.pair_dict = json.load(fp)
+                self.pair_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
         elif not self.follow_mode:
             logger.info("Could not find existing datadrawer, starting from scratch")
         else:
@@ -110,7 +143,19 @@ class FreqaiDataDrawer:
                 "sending null values back to strategy"
             )
 
-        return exists
+    def load_metric_tracker_from_disk(self):
+        """
+        Tries to load an existing metrics dictionary if the user
+        wants to collect metrics.
+        """
+        if self.freqai_info.get('write_metrics_to_disk', False):
+            exists = self.metric_tracker_path.is_file()
+            if exists:
+                with open(self.metric_tracker_path, "r") as fp:
+                    self.metric_tracker = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
+                logger.info("Loading existing metric tracker from disk.")
+            else:
+                logger.info("Could not find existing metric tracker, starting from scratch")
 
     def load_historic_predictions_from_disk(self):
         """
@@ -146,7 +191,7 @@ class FreqaiDataDrawer:
 
     def save_historic_predictions_to_disk(self):
         """
-        Save data drawer full of all pair model metadata in present model folder.
+        Save historic predictions pickle to disk
         """
         with open(self.historic_predictions_path, "wb") as fp:
             cloudpickle.dump(self.historic_predictions, fp, protocol=cloudpickle.DEFAULT_PROTOCOL)
@@ -154,6 +199,15 @@ class FreqaiDataDrawer:
         # create a backup
         shutil.copy(self.historic_predictions_path, self.historic_predictions_bkp_path)
 
+    def save_metric_tracker_to_disk(self):
+        """
+        Save metric tracker of all pair metrics collected.
+        """
+        with self.save_lock:
+            with open(self.metric_tracker_path, 'w') as fp:
+                rapidjson.dump(self.metric_tracker, fp, default=self.np_encoder,
+                               number_mode=rapidjson.NM_NATIVE)
+
     def save_drawer_to_disk(self):
         """
         Save data drawer full of all pair model metadata in present model folder.
@@ -257,7 +311,7 @@ class FreqaiDataDrawer:
 
     def append_model_predictions(self, pair: str, predictions: DataFrame,
                                  do_preds: NDArray[np.int_],
-                                 dk: FreqaiDataKitchen, len_df: int) -> None:
+                                 dk: FreqaiDataKitchen, strat_df: DataFrame) -> None:
         """
         Append model predictions to historic predictions dataframe, then set the
         strategy return dataframe to the tail of the historic predictions. The length of
@@ -266,6 +320,7 @@ class FreqaiDataDrawer:
         historic predictions.
         """
 
+        len_df = len(strat_df)
         index = self.historic_predictions[pair].index[-1:]
         columns = self.historic_predictions[pair].columns
 
@@ -293,6 +348,15 @@ class FreqaiDataDrawer:
             for return_str in rets:
                 df[return_str].iloc[-1] = rets[return_str]
 
+        # this logic carries users between version without needing to
+        # change their identifier
+        if 'close_price' not in df.columns:
+            df['close_price'] = np.nan
+            df['date_pred'] = np.nan
+
+        df['close_price'].iloc[-1] = strat_df['close'].iloc[-1]
+        df['date_pred'].iloc[-1] = strat_df['date'].iloc[-1]
+
         self.model_return_values[pair] = df.tail(len_df).reset_index(drop=True)
 
     def attach_return_values_to_return_dataframe(
@@ -402,9 +466,8 @@ class FreqaiDataDrawer:
     def save_data(self, model: Any, coin: str, dk: FreqaiDataKitchen) -> None:
         """
         Saves all data associated with a model for a single sub-train time range
-        :params:
-        :model: User trained model which can be reused for inferencing to generate
-        predictions
+        :param model: User trained model which can be reused for inferencing to generate
+                      predictions
         """
 
         if not dk.data_path.is_dir():
@@ -423,7 +486,7 @@ class FreqaiDataDrawer:
 
         dk.data["data_path"] = str(dk.data_path)
         dk.data["model_filename"] = str(dk.model_filename)
-        dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns)
+        dk.data["training_features_list"] = dk.training_features_list
         dk.data["label_list"] = dk.label_list
         # store the metadata
         with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp:
@@ -444,9 +507,14 @@ class FreqaiDataDrawer:
             )
 
         # if self.live:
+        # store as much in ram as possible to increase performance
         self.model_dictionary[coin] = model
         self.pair_dict[coin]["model_filename"] = dk.model_filename
         self.pair_dict[coin]["data_path"] = str(dk.data_path)
+        if coin not in self.meta_data_dictionary:
+            self.meta_data_dictionary[coin] = {}
+        self.meta_data_dictionary[coin]["train_df"] = dk.data_dictionary["train_features"]
+        self.meta_data_dictionary[coin]["meta_data"] = dk.data
         self.save_drawer_to_disk()
 
         return
@@ -457,7 +525,7 @@ class FreqaiDataDrawer:
         presaved backtesting (prediction file loading).
         """
         with open(dk.data_path / f"{dk.model_filename}_metadata.json", "r") as fp:
-            dk.data = json.load(fp)
+            dk.data = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
             dk.training_features_list = dk.data["training_features_list"]
             dk.label_list = dk.data["label_list"]
 
@@ -483,14 +551,19 @@ class FreqaiDataDrawer:
                     / dk.data_path.parts[-1]
                 )
 
-        with open(dk.data_path / f"{dk.model_filename}_metadata.json", "r") as fp:
-            dk.data = json.load(fp)
-            dk.training_features_list = dk.data["training_features_list"]
-            dk.label_list = dk.data["label_list"]
+        if coin in self.meta_data_dictionary:
+            dk.data = self.meta_data_dictionary[coin]["meta_data"]
+            dk.data_dictionary["train_features"] = self.meta_data_dictionary[coin]["train_df"]
+        else:
+            with open(dk.data_path / f"{dk.model_filename}_metadata.json", "r") as fp:
+                dk.data = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
 
-        dk.data_dictionary["train_features"] = pd.read_pickle(
-            dk.data_path / f"{dk.model_filename}_trained_df.pkl"
-        )
+            dk.data_dictionary["train_features"] = pd.read_pickle(
+                dk.data_path / f"{dk.model_filename}_trained_df.pkl"
+            )
+
+        dk.training_features_list = dk.data["training_features_list"]
+        dk.label_list = dk.data["label_list"]
 
         # try to access model in memory instead of loading object from disk to save time
         if dk.live and coin in self.model_dictionary:
@@ -522,8 +595,7 @@ class FreqaiDataDrawer:
         Append new candles to our stores historic data (in memory) so that
         we do not need to load candle history from disk and we dont need to
         pinging exchange multiple times for the same candle.
-        :params:
-        dataframe: DataFrame = strategy provided dataframe
+        :param dataframe: DataFrame = strategy provided dataframe
         """
         feat_params = self.freqai_info["feature_parameters"]
         with self.history_lock:
@@ -565,13 +637,14 @@ class FreqaiDataDrawer:
                         axis=0,
                     )
 
+            self.current_candle = history_data[dk.pair][self.config['timeframe']].iloc[-1]['date']
+
     def load_all_pair_histories(self, timerange: TimeRange, dk: FreqaiDataKitchen) -> None:
         """
         Load pair histories for all whitelist and corr_pairlist pairs.
         Only called once upon startup of bot.
-        :params:
-        timerange: TimeRange = full timerange required to populate all indicators
-        for training according to user defined train_period_days
+        :param timerange: TimeRange = full timerange required to populate all indicators
+                          for training according to user defined train_period_days
         """
         history_data = self.historic_data
 
@@ -594,10 +667,9 @@ class FreqaiDataDrawer:
         """
         Searches through our historic_data in memory and returns the dataframes relevant
         to the present pair.
-        :params:
-        timerange: TimeRange = full timerange required to populate all indicators
-        for training according to user defined train_period_days
-        metadata: dict = strategy furnished pair metadata
+        :param timerange: TimeRange = full timerange required to populate all indicators
+                          for training according to user defined train_period_days
+        :param metadata: dict = strategy furnished pair metadata
         """
         with self.history_lock:
             corr_dataframes: Dict[Any, Any] = {}
@@ -608,7 +680,8 @@ class FreqaiDataDrawer:
             )
 
             for tf in self.freqai_info["feature_parameters"].get("include_timeframes"):
-                base_dataframes[tf] = dk.slice_dataframe(timerange, historic_data[pair][tf])
+                base_dataframes[tf] = dk.slice_dataframe(
+                    timerange, historic_data[pair][tf]).reset_index(drop=True)
                 if pairs:
                     for p in pairs:
                         if pair in p:
@@ -617,25 +690,6 @@ class FreqaiDataDrawer:
                             corr_dataframes[p] = {}
                         corr_dataframes[p][tf] = dk.slice_dataframe(
                             timerange, historic_data[p][tf]
-                        )
+                        ).reset_index(drop=True)
 
         return corr_dataframes, base_dataframes
-
-    # to be used if we want to send predictions directly to the follower instead of forcing
-    # follower to load models and inference
-    # def save_model_return_values_to_disk(self) -> None:
-    #     with open(self.full_path / str('model_return_values.json'), "w") as fp:
-    #         json.dump(self.model_return_values, fp, default=self.np_encoder)
-
-    # def load_model_return_values_from_disk(self, dk: FreqaiDataKitchen) -> FreqaiDataKitchen:
-    #     exists = Path(self.full_path / str('model_return_values.json')).resolve().exists()
-    #     if exists:
-    #         with open(self.full_path / str('model_return_values.json'), "r") as fp:
-    #             self.model_return_values = json.load(fp)
-    #     elif not self.follow_mode:
-    #         logger.info("Could not find existing datadrawer, starting from scratch")
-    #     else:
-    #         logger.warning(f'Follower could not find pair_dictionary at {self.full_path} '
-    #                        'sending null values back to strategy')
-
-    #     return exists, dk

freqtrade/freqai/data_kitchen.py

@@ -1,7 +1,7 @@
 import copy
 import logging
 import shutil
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from math import cos, sin
 from pathlib import Path
 from typing import Any, Dict, List, Tuple
@@ -19,6 +19,7 @@ from sklearn.neighbors import NearestNeighbors
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import Config
+from freqtrade.data.converter import reduce_dataframe_footprint
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_seconds
 from freqtrade.strategy.interface import IStrategy
@@ -80,25 +81,32 @@ class FreqaiDataKitchen:
         self.svm_model: linear_model.SGDOneClassSVM = None
         self.keras: bool = self.freqai_config.get("keras", False)
         self.set_all_pairs()
-        if not self.live:
-            if not self.config["timerange"]:
-                raise OperationalException(
-                    'Please pass --timerange if you intend to use FreqAI for backtesting.')
-            self.full_timerange = self.create_fulltimerange(
-                self.config["timerange"], self.freqai_config.get("train_period_days", 0)
-            )
+        self.backtest_live_models = config.get("freqai_backtest_live_models", False)
 
-            (self.training_timeranges, self.backtesting_timeranges) = self.split_timerange(
-                self.full_timerange,
-                config["freqai"]["train_period_days"],
-                config["freqai"]["backtest_period_days"],
-            )
+        if not self.live:
+            self.full_path = self.get_full_models_path(self.config)
+
+            if self.backtest_live_models:
+                if self.pair:
+                    self.set_timerange_from_ready_models()
+                    (self.training_timeranges,
+                     self.backtesting_timeranges) = self.split_timerange_live_models()
+            else:
+                self.full_timerange = self.create_fulltimerange(
+                    self.config["timerange"], self.freqai_config.get("train_period_days", 0)
+                )
+                (self.training_timeranges, self.backtesting_timeranges) = self.split_timerange(
+                    self.full_timerange,
+                    config["freqai"]["train_period_days"],
+                    config["freqai"]["backtest_period_days"],
+                )
 
         self.data['extra_returns_per_train'] = self.freqai_config.get('extra_returns_per_train', {})
         self.thread_count = self.freqai_config.get("data_kitchen_thread_count", -1)
         self.train_dates: DataFrame = pd.DataFrame()
         self.unique_classes: Dict[str, list] = {}
         self.unique_class_list: list = []
+        self.backtest_live_models_data: Dict[str, Any] = {}
 
     def set_paths(
         self,
@@ -107,14 +115,10 @@ class FreqaiDataKitchen:
     ) -> None:
         """
         Set the paths to the data for the present coin/botloop
-        :params:
-        metadata: dict = strategy furnished pair metadata
-        trained_timestamp: int = timestamp of most recent training
+        :param metadata: dict = strategy furnished pair metadata
+        :param trained_timestamp: int = timestamp of most recent training
         """
-        self.full_path = Path(
-            self.config["user_data_dir"] / "models" / str(self.freqai_config.get("identifier"))
-        )
-
+        self.full_path = self.get_full_models_path(self.config)
         self.data_path = Path(
             self.full_path
             / f"sub-train-{pair.split('/')[0]}_{trained_timestamp}"
@@ -129,25 +133,20 @@ class FreqaiDataKitchen:
         Given the dataframe for the full history for training, split the data into
         training and test data according to user specified parameters in configuration
         file.
-        :filtered_dataframe: cleaned dataframe ready to be split.
-        :labels: cleaned labels ready to be split.
+        :param filtered_dataframe: cleaned dataframe ready to be split.
+        :param labels: cleaned labels ready to be split.
         """
         feat_dict = self.freqai_config["feature_parameters"]
 
+        if 'shuffle' not in self.freqai_config['data_split_parameters']:
+            self.freqai_config["data_split_parameters"].update({'shuffle': False})
+
         weights: npt.ArrayLike
         if feat_dict.get("weight_factor", 0) > 0:
             weights = self.set_weights_higher_recent(len(filtered_dataframe))
         else:
             weights = np.ones(len(filtered_dataframe))
 
-        if feat_dict.get("stratify_training_data", 0) > 0:
-            stratification = np.zeros(len(filtered_dataframe))
-            for i in range(1, len(stratification)):
-                if i % feat_dict.get("stratify_training_data", 0) == 0:
-                    stratification[i] = 1
-        else:
-            stratification = None
-
         if self.freqai_config.get('data_split_parameters', {}).get('test_size', 0.1) != 0:
             (
                 train_features,
@@ -160,7 +159,6 @@ class FreqaiDataKitchen:
                 filtered_dataframe[: filtered_dataframe.shape[0]],
                 labels,
                 weights,
-                stratify=stratification,
                 **self.config["freqai"]["data_split_parameters"],
             )
         else:
@@ -195,13 +193,14 @@ class FreqaiDataKitchen:
         remove all NaNs. Any row with a NaN is removed from training dataset or replaced with
         0s in the prediction dataset. However, prediction dataset do_predict will reflect any
         row that had a NaN and will shield user from that prediction.
-        :params:
-        :unfiltered_df: the full dataframe for the present training period
-        :training_feature_list: list, the training feature list constructed by
-        self.build_feature_list() according to user specified parameters in the configuration file.
-        :labels: the labels for the dataset
-        :training_filter: boolean which lets the function know if it is training data or
-        prediction data to be filtered.
+
+        :param unfiltered_df: the full dataframe for the present training period
+        :param training_feature_list: list, the training feature list constructed by
+                                      self.build_feature_list() according to user specified
+                                      parameters in the configuration file.
+        :param labels: the labels for the dataset
+        :param training_filter: boolean which lets the function know if it is training data or
+                                prediction data to be filtered.
         :returns:
         :filtered_df: dataframe cleaned of NaNs and only containing the user
         requested feature set.
@@ -210,18 +209,21 @@ class FreqaiDataKitchen:
         filtered_df = unfiltered_df.filter(training_feature_list, axis=1)
         filtered_df = filtered_df.replace([np.inf, -np.inf], np.nan)
 
-        drop_index = pd.isnull(filtered_df).any(1)  # get the rows that have NaNs,
+        drop_index = pd.isnull(filtered_df).any(axis=1)  # get the rows that have NaNs,
         drop_index = drop_index.replace(True, 1).replace(False, 0)  # pep8 requirement.
         if (training_filter):
             const_cols = list((filtered_df.nunique() == 1).loc[lambda x: x].index)
             if const_cols:
                 filtered_df = filtered_df.filter(filtered_df.columns.difference(const_cols))
+                self.data['constant_features_list'] = const_cols
                 logger.warning(f"Removed features {const_cols} with constant values.")
+            else:
+                self.data['constant_features_list'] = []
             # we don't care about total row number (total no. datapoints) in training, we only care
             # about removing any row with NaNs
             # if labels has multiple columns (user wants to train multiple modelEs), we detect here
             labels = unfiltered_df.filter(label_list, axis=1)
-            drop_index_labels = pd.isnull(labels).any(1)
+            drop_index_labels = pd.isnull(labels).any(axis=1)
             drop_index_labels = drop_index_labels.replace(True, 1).replace(False, 0)
             dates = unfiltered_df['date']
             filtered_df = filtered_df[
@@ -247,9 +249,11 @@ class FreqaiDataKitchen:
             self.data["filter_drop_index_training"] = drop_index
 
         else:
+            if 'constant_features_list' in self.data and len(self.data['constant_features_list']):
+                filtered_df = self.check_pred_labels(filtered_df)
             # we are backtesting so we need to preserve row number to send back to strategy,
             # so now we use do_predict to avoid any prediction based on a NaN
-            drop_index = pd.isnull(filtered_df).any(1)
+            drop_index = pd.isnull(filtered_df).any(axis=1)
             self.data["filter_drop_index_prediction"] = drop_index
             filtered_df.fillna(0, inplace=True)
             # replacing all NaNs with zeros to avoid issues in 'prediction', but any prediction
@@ -291,8 +295,8 @@ class FreqaiDataKitchen:
     def normalize_data(self, data_dictionary: Dict) -> Dict[Any, Any]:
         """
         Normalize all data in the data_dictionary according to the training dataset
-        :params:
-        :data_dictionary: dictionary containing the cleaned and split training/test data/labels
+        :param data_dictionary: dictionary containing the cleaned and
+                                split training/test data/labels
         :returns:
         :data_dictionary: updated dictionary with standardized values.
         """
@@ -355,13 +359,19 @@ class FreqaiDataKitchen:
         :param df: Dataframe to be standardized
         """
 
-        for item in df.keys():
-            df[item] = (
-                2
-                * (df[item] - self.data[f"{item}_min"])
-                / (self.data[f"{item}_max"] - self.data[f"{item}_min"])
-                - 1
-            )
+        train_max = [None] * len(df.keys())
+        train_min = [None] * len(df.keys())
+
+        for i, item in enumerate(df.keys()):
+            train_max[i] = self.data[f"{item}_max"]
+            train_min[i] = self.data[f"{item}_min"]
+
+        train_max_series = pd.Series(train_max, index=df.keys())
+        train_min_series = pd.Series(train_min, index=df.keys())
+
+        df = (
+            2 * (df - train_min_series) / (train_max_series - train_min_series) - 1
+        )
 
         return df
 
@@ -423,9 +433,7 @@ class FreqaiDataKitchen:
             timerange_train.stopts = timerange_train.startts + train_period_days
 
             first = False
-            start = datetime.fromtimestamp(timerange_train.startts, tz=timezone.utc)
-            stop = datetime.fromtimestamp(timerange_train.stopts, tz=timezone.utc)
-            tr_training_list.append(start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d"))
+            tr_training_list.append(timerange_train.timerange_str)
             tr_training_list_timerange.append(copy.deepcopy(timerange_train))
 
             # associated backtest period
@@ -437,9 +445,7 @@ class FreqaiDataKitchen:
             if timerange_backtest.stopts > config_timerange.stopts:
                 timerange_backtest.stopts = config_timerange.stopts
 
-            start = datetime.fromtimestamp(timerange_backtest.startts, tz=timezone.utc)
-            stop = datetime.fromtimestamp(timerange_backtest.stopts, tz=timezone.utc)
-            tr_backtesting_list.append(start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d"))
+            tr_backtesting_list.append(timerange_backtest.timerange_str)
             tr_backtesting_list_timerange.append(copy.deepcopy(timerange_backtest))
 
             # ensure we are predicting on exactly same amount of data as requested by user defined
@@ -450,6 +456,29 @@ class FreqaiDataKitchen:
         # print(tr_training_list, tr_backtesting_list)
         return tr_training_list_timerange, tr_backtesting_list_timerange
 
+    def split_timerange_live_models(
+        self
+    ) -> Tuple[list, list]:
+
+        tr_backtesting_list_timerange = []
+        asset = self.pair.split("/")[0]
+        if asset not in self.backtest_live_models_data["assets_end_dates"]:
+            raise OperationalException(
+                f"Model not available for pair {self.pair}. "
+                "Please, try again after removing this pair from the configuration file."
+            )
+        asset_data = self.backtest_live_models_data["assets_end_dates"][asset]
+        backtesting_timerange = self.backtest_live_models_data["backtesting_timerange"]
+        model_end_dates = [x for x in asset_data]
+        model_end_dates.append(backtesting_timerange.stopts)
+        model_end_dates.sort()
+        for index, item in enumerate(model_end_dates):
+            if len(model_end_dates) > (index + 1):
+                tr_to_add = TimeRange("date", "date", item, model_end_dates[index + 1])
+                tr_backtesting_list_timerange.append(tr_to_add)
+
+        return tr_backtesting_list_timerange, tr_backtesting_list_timerange
+
     def slice_dataframe(self, timerange: TimeRange, df: DataFrame) -> DataFrame:
         """
         Given a full dataframe, extract the user desired window
@@ -458,14 +487,28 @@ class FreqaiDataKitchen:
                    it is sliced down to just the present training period.
         """
 
-        start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
-        stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
-        df = df.loc[df["date"] >= start, :]
+        df = df.loc[df["date"] >= timerange.startdt, :]
         if not self.live:
-            df = df.loc[df["date"] < stop, :]
+            df = df.loc[df["date"] < timerange.stopdt, :]
 
         return df
 
+    def check_pred_labels(self, df_predictions: DataFrame) -> DataFrame:
+        """
+        Check that prediction feature labels match training feature labels.
+        :param df_predictions: incoming predictions
+        """
+        constant_labels = self.data['constant_features_list']
+        df_predictions = df_predictions.filter(
+            df_predictions.columns.difference(constant_labels)
+        )
+        logger.warning(
+            f"Removed {len(constant_labels)} features from prediction features, "
+            f"these were considered constant values during most recent training."
+        )
+
+        return df_predictions
+
     def principal_component_analysis(self) -> None:
         """
         Performs Principal Component Analysis on the data for dimensionality reduction
@@ -522,8 +565,7 @@ class FreqaiDataKitchen:
     def pca_transform(self, filtered_dataframe: DataFrame) -> None:
         """
         Use an existing pca transform to transform data into components
-        :params:
-        filtered_dataframe: DataFrame = the cleaned dataframe
+        :param filtered_dataframe: DataFrame = the cleaned dataframe
         """
         pca_components = self.pca.transform(filtered_dataframe)
         self.data_dictionary["prediction_features"] = pd.DataFrame(
@@ -567,8 +609,7 @@ class FreqaiDataKitchen:
         """
         Build/inference a Support Vector Machine to detect outliers
         in training data and prediction
-        :params:
-        predict: bool = If true, inference an existing SVM model, else construct one
+        :param predict: bool = If true, inference an existing SVM model, else construct one
         """
 
         if self.keras:
@@ -653,11 +694,11 @@ class FreqaiDataKitchen:
         Use DBSCAN to cluster training data and remove "noisy" data (read outliers).
         User controls this via the config param `DBSCAN_outlier_pct` which indicates the
         pct of training data that they want to be considered outliers.
-        :params:
-        predict: bool = If False (training), iterate to find the best hyper parameters to match
-        user requested outlier percent target. If True (prediction), use the parameters
-        determined from the previous training to estimate if the current prediction point
-        is an outlier.
+        :param predict: bool = If False (training), iterate to find the best hyper parameters
+                        to match user requested outlier percent target.
+                        If True (prediction), use the parameters determined from
+                        the previous training to estimate if the current prediction point
+                        is an outlier.
         """
 
         if predict:
@@ -808,7 +849,7 @@ class FreqaiDataKitchen:
                 :, :no_prev_pts
             ]
         distances = distances.replace([np.inf, -np.inf], np.nan)
-        drop_index = pd.isnull(distances).any(1)
+        drop_index = pd.isnull(distances).any(axis=1)
         distances = distances[drop_index == 0]
 
         inliers = pd.DataFrame(index=distances.index)
@@ -881,6 +922,7 @@ class FreqaiDataKitchen:
         """
         column_names = dataframe.columns
         features = [c for c in column_names if "%" in c]
+
         if not features:
             raise OperationalException("Could not find any features!")
 
@@ -942,8 +984,13 @@ class FreqaiDataKitchen:
             append_df[label] = predictions[label]
             if append_df[label].dtype == object:
                 continue
-            append_df[f"{label}_mean"] = self.data["labels_mean"][label]
-            append_df[f"{label}_std"] = self.data["labels_std"][label]
+            if "labels_mean" in self.data:
+                append_df[f"{label}_mean"] = self.data["labels_mean"][label]
+            if "labels_std" in self.data:
+                append_df[f"{label}_std"] = self.data["labels_std"][label]
+
+        for extra_col in self.data["extra_returns_per_train"]:
+            append_df[f"{extra_col}"] = self.data["extra_returns_per_train"][extra_col]
 
         append_df["do_predict"] = do_predict
         if self.freqai_config["feature_parameters"].get("DI_threshold", 0) > 0:
@@ -1005,14 +1052,7 @@ class FreqaiDataKitchen:
         backtest_timerange.startts = (
             backtest_timerange.startts - backtest_period_days * SECONDS_IN_DAY
         )
-        start = datetime.fromtimestamp(backtest_timerange.startts, tz=timezone.utc)
-        stop = datetime.fromtimestamp(backtest_timerange.stopts, tz=timezone.utc)
-        full_timerange = start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d")
-
-        self.full_path = Path(
-            self.config["user_data_dir"] / "models" / f"{self.freqai_config['identifier']}"
-        )
-
+        full_timerange = backtest_timerange.timerange_str
         config_path = Path(self.config["config_files"][0])
 
         if not self.full_path.is_dir():
@@ -1095,15 +1135,15 @@ class FreqaiDataKitchen:
 
         return retrain, trained_timerange, data_load_timerange
 
-    def set_new_model_names(self, pair: str, trained_timerange: TimeRange):
+    def set_new_model_names(self, pair: str, timestamp_id: int):
 
         coin, _ = pair.split("/")
         self.data_path = Path(
             self.full_path
-            / f"sub-train-{pair.split('/')[0]}_{int(trained_timerange.stopts)}"
+            / f"sub-train-{pair.split('/')[0]}_{timestamp_id}"
         )
 
-        self.model_filename = f"cb_{coin.lower()}_{int(trained_timerange.stopts)}"
+        self.model_filename = f"cb_{coin.lower()}_{timestamp_id}"
 
     def set_all_pairs(self) -> None:
 
@@ -1114,6 +1154,54 @@ class FreqaiDataKitchen:
             if pair not in self.all_pairs:
                 self.all_pairs.append(pair)
 
+    def extract_corr_pair_columns_from_populated_indicators(
+        self,
+        dataframe: DataFrame
+    ) -> Dict[str, DataFrame]:
+        """
+        Find the columns of the dataframe corresponding to the corr_pairlist, save them
+        in a dictionary to be reused and attached to other pairs.
+
+        :param dataframe: fully populated dataframe (current pair + corr_pairs)
+        :return: corr_dataframes, dictionary of dataframes to be attached
+                 to other pairs in same candle.
+        """
+        corr_dataframes: Dict[str, DataFrame] = {}
+        pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
+
+        for pair in pairs:
+            pair = pair.replace(':', '')  # lightgbm doesnt like colons
+            valid_strs = [f"%-{pair}", f"%{pair}", f"%_{pair}"]
+            pair_cols = [col for col in dataframe.columns if
+                         any(substr in col for substr in valid_strs)]
+            if pair_cols:
+                pair_cols.insert(0, 'date')
+                corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1)
+
+        return corr_dataframes
+
+    def attach_corr_pair_columns(self, dataframe: DataFrame,
+                                 corr_dataframes: Dict[str, DataFrame],
+                                 current_pair: str) -> DataFrame:
+        """
+        Attach the existing corr_pair dataframes to the current pair dataframe before training
+
+        :param dataframe: current pair strategy dataframe, indicators populated already
+        :param corr_dataframes: dictionary of saved dataframes from earlier in the same candle
+        :param current_pair: current pair to which we will attach corr pair dataframe
+        :return:
+        :dataframe: current pair dataframe of populated indicators, concatenated with corr_pairs
+                    ready for training
+        """
+        pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
+        current_pair = current_pair.replace(':', '')
+        for pair in pairs:
+            pair = pair.replace(':', '')  # lightgbm doesnt work with colons
+            if current_pair != pair:
+                dataframe = dataframe.merge(corr_dataframes[pair], how='left', on='date')
+
+        return dataframe
+
     def use_strategy_to_populate_indicators(
         self,
         strategy: IStrategy,
@@ -1121,26 +1209,25 @@ class FreqaiDataKitchen:
         base_dataframes: dict = {},
         pair: str = "",
         prediction_dataframe: DataFrame = pd.DataFrame(),
+        do_corr_pairs: bool = True,
     ) -> DataFrame:
         """
-        Use the user defined strategy for populating indicators during
-        retrain
-        :params:
-        strategy: IStrategy = user defined strategy object
-        corr_dataframes: dict = dict containing the informative pair dataframes
-        (for user defined timeframes)
-        base_dataframes: dict = dict containing the current pair dataframes
-        (for user defined timeframes)
-        metadata: dict = strategy furnished pair metadata
-        :returns:
+        Use the user defined strategy for populating indicators during retrain
+        :param strategy: IStrategy = user defined strategy object
+        :param corr_dataframes: dict = dict containing the informative pair dataframes
+                                (for user defined timeframes)
+        :param base_dataframes: dict = dict containing the current pair dataframes
+                                (for user defined timeframes)
+        :param metadata: dict = strategy furnished pair metadata
+        :return:
         dataframe: DataFrame = dataframe containing populated indicators
         """
 
         # for prediction dataframe creation, we let dataprovider handle everything in the strategy
         # so we create empty dictionaries, which allows us to pass None to
         # `populate_any_indicators()`. Signaling we want the dp to give us the live dataframe.
-        tfs = self.freqai_config["feature_parameters"].get("include_timeframes")
-        pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
+        tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes")
+        pairs: List[str] = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
         if not prediction_dataframe.empty:
             dataframe = prediction_dataframe.copy()
             for tf in tfs:
@@ -1163,19 +1250,27 @@ class FreqaiDataKitchen:
                 informative=base_dataframes[tf],
                 set_generalized_indicators=sgi
             )
-            if pairs:
-                for i in pairs:
-                    if pair in i:
-                        continue  # dont repeat anything from whitelist
+
+        # ensure corr pairs are always last
+        for corr_pair in pairs:
+            if pair == corr_pair:
+                continue  # dont repeat anything from whitelist
+            for tf in tfs:
+                if pairs and do_corr_pairs:
                     dataframe = strategy.populate_any_indicators(
-                        i,
+                        corr_pair,
                         dataframe.copy(),
                         tf,
-                        informative=corr_dataframes[i][tf]
+                        informative=corr_dataframes[corr_pair][tf]
                     )
 
         self.get_unique_classes_from_labels(dataframe)
 
+        dataframe = self.remove_special_chars_from_feature_names(dataframe)
+
+        if self.config.get('reduce_df_footprint', False):
+            dataframe = reduce_dataframe_footprint(dataframe)
+
         return dataframe
 
     def fit_labels(self) -> None:
@@ -1242,14 +1337,16 @@ class FreqaiDataKitchen:
         append_df = pd.read_hdf(self.backtesting_results_path)
         return append_df
 
-    def check_if_backtest_prediction_exists(
-        self
+    def check_if_backtest_prediction_is_valid(
+        self,
+        len_backtest_df: int
     ) -> bool:
         """
-        Check if a backtesting prediction already exists
-        :param dk: FreqaiDataKitchen
+        Check if a backtesting prediction already exists and if the predictions
+        to append have the same size as the backtesting dataframe slice
+        :param length_backtesting_dataframe: Length of backtesting dataframe slice
         :return:
-        :boolean: whether the prediction file exists or not.
+        :boolean: whether the prediction file is valid.
         """
         path_to_predictionfile = Path(self.full_path /
                                       self.backtest_predictions_folder /
@@ -1257,10 +1354,134 @@ class FreqaiDataKitchen:
         self.backtesting_results_path = path_to_predictionfile
 
         file_exists = path_to_predictionfile.is_file()
+
         if file_exists:
-            logger.info(f"Found backtesting prediction file at {path_to_predictionfile}")
+            append_df = self.get_backtesting_prediction()
+            if len(append_df) == len_backtest_df:
+                logger.info(f"Found backtesting prediction file at {path_to_predictionfile}")
+                return True
+            else:
+                logger.info("A new backtesting prediction file is required. "
+                            "(Number of predictions is different from dataframe length).")
+                return False
         else:
             logger.info(
                 f"Could not find backtesting prediction file at {path_to_predictionfile}"
             )
-        return file_exists
+            return False
+
+    def set_timerange_from_ready_models(self):
+        backtesting_timerange, \
+            assets_end_dates = (
+                self.get_timerange_and_assets_end_dates_from_ready_models(self.full_path))
+
+        self.backtest_live_models_data = {
+            "backtesting_timerange": backtesting_timerange,
+            "assets_end_dates": assets_end_dates
+            }
+        return
+
+    def get_full_models_path(self, config: Config) -> Path:
+        """
+        Returns default FreqAI model path
+        :param config: Configuration dictionary
+        """
+        freqai_config: Dict[str, Any] = config["freqai"]
+        return Path(
+            config["user_data_dir"] / "models" / str(freqai_config.get("identifier"))
+        )
+
+    def get_timerange_and_assets_end_dates_from_ready_models(
+            self, models_path: Path) -> Tuple[TimeRange, Dict[str, Any]]:
+        """
+        Returns timerange information based on a FreqAI model directory
+        :param models_path: FreqAI model path
+
+        :return: a Tuple with (Timerange calculated from directory and
+        a Dict with pair and model end training dates info)
+        """
+        all_models_end_dates = []
+        assets_end_dates: Dict[str, Any] = self.get_assets_timestamps_training_from_ready_models(
+                                           models_path)
+        for key in assets_end_dates:
+            for model_end_date in assets_end_dates[key]:
+                if model_end_date not in all_models_end_dates:
+                    all_models_end_dates.append(model_end_date)
+
+        if len(all_models_end_dates) == 0:
+            raise OperationalException(
+                'At least 1 saved model is required to '
+                'run backtest with the freqai-backtest-live-models option'
+            )
+
+        if len(all_models_end_dates) == 1:
+            logger.warning(
+                "Only 1 model was found. Backtesting will run with the "
+                "timerange from the end of the training date to the current date"
+            )
+
+        finish_timestamp = int(datetime.now(tz=timezone.utc).timestamp())
+        if len(all_models_end_dates) > 1:
+            # After last model end date, use the same period from previous model
+            # to finish the backtest
+            all_models_end_dates.sort(reverse=True)
+            finish_timestamp = all_models_end_dates[0] + \
+                (all_models_end_dates[0] - all_models_end_dates[1])
+
+        all_models_end_dates.append(finish_timestamp)
+        all_models_end_dates.sort()
+        start_date = (datetime(*datetime.fromtimestamp(min(all_models_end_dates),
+                      timezone.utc).timetuple()[:3], tzinfo=timezone.utc))
+        end_date = (datetime(*datetime.fromtimestamp(max(all_models_end_dates),
+                    timezone.utc).timetuple()[:3], tzinfo=timezone.utc))
+
+        # add 1 day to string timerange to ensure BT module will load all dataframe data
+        end_date = end_date + timedelta(days=1)
+        backtesting_timerange = TimeRange(
+            'date', 'date', int(start_date.timestamp()), int(end_date.timestamp())
+        )
+        return backtesting_timerange, assets_end_dates
+
+    def get_assets_timestamps_training_from_ready_models(
+            self, models_path: Path) -> Dict[str, Any]:
+        """
+        Scan the models path and returns all assets end training dates (timestamp)
+        :param models_path: FreqAI model path
+
+        :return: a Dict with asset and model end training dates info
+        """
+        assets_end_dates: Dict[str, Any] = {}
+        if not models_path.is_dir():
+            raise OperationalException(
+                'Model folders not found. Saved models are required '
+                'to run backtest with the freqai-backtest-live-models option'
+            )
+        for model_dir in models_path.iterdir():
+            if str(model_dir.name).startswith("sub-train"):
+                model_end_date = int(model_dir.name.split("_")[1])
+                asset = model_dir.name.split("_")[0].replace("sub-train-", "")
+                model_file_name = (
+                    f"cb_{str(model_dir.name).replace('sub-train-', '').lower()}"
+                    "_model.joblib"
+                )
+
+                model_path_file = Path(model_dir / model_file_name)
+                if model_path_file.is_file():
+                    if asset not in assets_end_dates:
+                        assets_end_dates[asset] = []
+                    assets_end_dates[asset].append(model_end_date)
+
+        return assets_end_dates
+
+    def remove_special_chars_from_feature_names(self, dataframe: pd.DataFrame) -> pd.DataFrame:
+        """
+        Remove all special characters from feature strings (:)
+        :param dataframe: the dataframe that just finished indicator population. (unfiltered)
+        :return: dataframe with cleaned featrue names
+        """
+
+        spec_chars = [':']
+        for c in spec_chars:
+            dataframe.columns = dataframe.columns.str.replace(c, "")
+
+        return dataframe

freqtrade/freqai/freqai_interface.py

@@ -1,13 +1,11 @@
 import logging
-import shutil
 import threading
 import time
 from abc import ABC, abstractmethod
 from collections import deque
 from datetime import datetime, timezone
 from pathlib import Path
-from threading import Lock
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, List, Literal, Tuple
 
 import numpy as np
 import pandas as pd
@@ -15,13 +13,13 @@ from numpy.typing import NDArray
 from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DATETIME_PRINT_FORMAT, Config
+from freqtrade.constants import Config
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_seconds
 from freqtrade.freqai.data_drawer import FreqaiDataDrawer
 from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
-from freqtrade.freqai.utils import plot_feature_importance
+from freqtrade.freqai.utils import plot_feature_importance, record_params
 from freqtrade.strategy.interface import IStrategy
 
 
@@ -61,6 +59,7 @@ class IFreqaiModel(ABC):
             "data_split_parameters", {})
         self.model_training_parameters: Dict[str, Any] = config.get("freqai", {}).get(
             "model_training_parameters", {})
+        self.identifier: str = self.freqai_info.get("identifier", "no_id_provided")
         self.retrain = False
         self.first = True
         self.set_full_path()
@@ -69,23 +68,23 @@ class IFreqaiModel(ABC):
         if self.save_backtest_models:
             logger.info('Backtesting module configured to save all models.')
         self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode)
-        self.identifier: str = self.freqai_info.get("identifier", "no_id_provided")
+        # set current candle to arbitrary historical date
+        self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc)
+        self.dd.current_candle = self.current_candle
         self.scanning = False
         self.ft_params = self.freqai_info["feature_parameters"]
+        self.corr_pairlist: List[str] = self.ft_params.get("include_corr_pairlist", [])
         self.keras: bool = self.freqai_info.get("keras", False)
         if self.keras and self.ft_params.get("DI_threshold", 0):
             self.ft_params["DI_threshold"] = 0
             logger.warning("DI threshold is not configured for Keras models yet. Deactivating.")
-        self.CONV_WIDTH = self.freqai_info.get("conv_width", 2)
+        self.CONV_WIDTH = self.freqai_info.get('conv_width', 1)
         if self.ft_params.get("inlier_metric_window", 0):
             self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2
         self.pair_it = 0
         self.pair_it_train = 0
         self.total_pairs = len(self.config.get("exchange", {}).get("pair_whitelist"))
         self.train_queue = self._set_train_queue()
-        self.last_trade_database_summary: DataFrame = {}
-        self.current_trade_database_summary: DataFrame = {}
-        self.analysis_lock = Lock()
         self.inference_time: float = 0
         self.train_time: float = 0
         self.begin_time: float = 0
@@ -93,10 +92,15 @@ class IFreqaiModel(ABC):
         self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe'])
         self.continual_learning = self.freqai_info.get('continual_learning', False)
         self.plot_features = self.ft_params.get("plot_feature_importances", 0)
-
+        self.corr_dataframes: Dict[str, DataFrame] = {}
+        # get_corr_dataframes is controlling the caching of corr_dataframes
+        # for improved performance. Careful with this boolean.
+        self.get_corr_dataframes: bool = True
         self._threads: List[threading.Thread] = []
         self._stop_event = threading.Event()
 
+        record_params(config, self.full_path)
+
     def __getstate__(self):
         """
         Return an empty state to be pickled in hyperopt
@@ -135,7 +139,11 @@ class IFreqaiModel(ABC):
         # the concatenated results for the full backtesting period back to the strategy.
         elif not self.follow_mode:
             self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
-            logger.info(f"Training {len(self.dk.training_timeranges)} timeranges")
+            if self.dk.backtest_live_models:
+                logger.info(
+                    f"Backtesting {len(self.dk.backtesting_timeranges)} timeranges (live models)")
+            else:
+                logger.info(f"Training {len(self.dk.training_timeranges)} timeranges")
             dataframe = self.dk.use_strategy_to_populate_indicators(
                 strategy, prediction_dataframe=dataframe, pair=metadata["pair"]
             )
@@ -144,7 +152,7 @@ class IFreqaiModel(ABC):
         dataframe = dk.remove_features_from_df(dk.return_dataframe)
         self.clean_up()
         if self.live:
-            self.inference_timer('stop')
+            self.inference_timer('stop', metadata["pair"])
         return dataframe
 
     def clean_up(self):
@@ -196,29 +204,31 @@ class IFreqaiModel(ABC):
             (_, trained_timestamp, _) = self.dd.get_pair_dict_info(pair)
 
             dk = FreqaiDataKitchen(self.config, self.live, pair)
-            dk.set_paths(pair, trained_timestamp)
             (
                 retrain,
                 new_trained_timerange,
                 data_load_timerange,
             ) = dk.check_if_new_training_required(trained_timestamp)
-            dk.set_paths(pair, new_trained_timerange.stopts)
 
             if retrain:
                 self.train_timer('start')
+                dk.set_paths(pair, new_trained_timerange.stopts)
                 try:
                     self.extract_data_and_train_model(
                         new_trained_timerange, pair, strategy, dk, data_load_timerange
                     )
                 except Exception as msg:
-                    logger.warning(f'Training {pair} raised exception {msg}, skipping.')
+                    logger.warning(f"Training {pair} raised exception {msg.__class__.__name__}. "
+                                   f"Message: {msg}, skipping.")
 
-                self.train_timer('stop')
+                self.train_timer('stop', pair)
 
                 # only rotate the queue after the first has been trained.
                 self.train_queue.rotate(-1)
 
                 self.dd.save_historic_predictions_to_disk()
+                if self.freqai_info.get('write_metrics_to_disk', False):
+                    self.dd.save_metric_tracker_to_disk()
 
     def start_backtesting(
         self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen
@@ -253,29 +263,23 @@ class IFreqaiModel(ABC):
             dataframe_train = dk.slice_dataframe(tr_train, dataframe)
             dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe)
 
-            trained_timestamp = tr_train
-            tr_train_startts_str = datetime.fromtimestamp(
-                                                tr_train.startts,
-                                                tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT)
-            tr_train_stopts_str = datetime.fromtimestamp(
-                                                tr_train.stopts,
-                                                tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT)
-            logger.info(
-                f"Training {pair}, {self.pair_it}/{self.total_pairs} pairs"
-                f" from {tr_train_startts_str} to {tr_train_stopts_str}, {train_it}/{total_trains} "
-                "trains"
-            )
+            if not self.ensure_data_exists(dataframe_backtest, tr_backtest, pair):
+                continue
 
-            trained_timestamp_int = int(trained_timestamp.stopts)
-            dk.data_path = Path(
-                dk.full_path / f"sub-train-{pair.split('/')[0]}_{trained_timestamp_int}"
-                )
+            self.log_backtesting_progress(tr_train, pair, train_it, total_trains)
 
-            dk.set_new_model_names(pair, trained_timestamp)
+            timestamp_model_id = int(tr_train.stopts)
+            if dk.backtest_live_models:
+                timestamp_model_id = int(tr_backtest.startts)
 
-            if dk.check_if_backtest_prediction_exists():
+            dk.set_paths(pair, timestamp_model_id)
+
+            dk.set_new_model_names(pair, timestamp_model_id)
+
+            if dk.check_if_backtest_prediction_is_valid(len(dataframe_backtest)):
                 self.dd.load_metadata(dk)
-                self.check_if_feature_list_matches_strategy(dataframe_train, dk)
+                dk.find_features(dataframe_train)
+                self.check_if_feature_list_matches_strategy(dk)
                 append_df = dk.get_backtesting_prediction()
                 dk.append_predictions(append_df)
             else:
@@ -284,7 +288,7 @@ class IFreqaiModel(ABC):
                     dk.find_labels(dataframe_train)
                     self.model = self.train(dataframe_train, pair, dk)
                     self.dd.pair_dict[pair]["trained_timestamp"] = int(
-                        trained_timestamp.stopts)
+                        tr_train.stopts)
                     if self.plot_features:
                         plot_feature_importance(self.model, pair, dk, self.plot_features)
                     if self.save_backtest_models:
@@ -296,7 +300,6 @@ class IFreqaiModel(ABC):
                 else:
                     self.model = self.dd.load_data(pair, dk)
 
-                # self.check_if_feature_list_matches_strategy(dataframe_train, dk)
                 pred_df, do_preds = self.predict(dataframe_backtest, dk)
                 append_df = dk.get_predictions_to_append(pred_df, do_preds)
                 dk.append_predictions(append_df)
@@ -337,6 +340,7 @@ class IFreqaiModel(ABC):
         if self.dd.historic_data:
             self.dd.update_historic_data(strategy, dk)
             logger.debug(f'Updating historic data on pair {metadata["pair"]}')
+            self.track_current_candle()
 
         if not self.follow_mode:
 
@@ -363,10 +367,10 @@ class IFreqaiModel(ABC):
         # load the model and associated data into the data kitchen
         self.model = self.dd.load_data(metadata["pair"], dk)
 
-        with self.analysis_lock:
-            dataframe = self.dk.use_strategy_to_populate_indicators(
-                strategy, prediction_dataframe=dataframe, pair=metadata["pair"]
-            )
+        dataframe = dk.use_strategy_to_populate_indicators(
+            strategy, prediction_dataframe=dataframe, pair=metadata["pair"],
+            do_corr_pairs=self.get_corr_dataframes
+        )
 
         if not self.model:
             logger.warning(
@@ -375,6 +379,9 @@ class IFreqaiModel(ABC):
             self.dd.return_null_values_to_strategy(dataframe, dk)
             return dk
 
+        if self.corr_pairlist:
+            dataframe = self.cache_corr_pairlist_dfs(dataframe, dk)
+
         dk.find_labels(dataframe)
 
         self.build_strategy_return_arrays(dataframe, dk, metadata["pair"], trained_timestamp)
@@ -393,7 +400,7 @@ class IFreqaiModel(ABC):
             # allows FreqUI to show full return values.
             pred_df, do_preds = self.predict(dataframe, dk)
             if pair not in self.dd.historic_predictions:
-                self.set_initial_historic_predictions(pred_df, dk, pair)
+                self.set_initial_historic_predictions(pred_df, dk, pair, dataframe)
             self.dd.set_initial_return_values(pair, pred_df)
 
             dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
@@ -414,13 +421,13 @@ class IFreqaiModel(ABC):
 
         if self.freqai_info.get('fit_live_predictions_candles', 0) and self.live:
             self.fit_live_predictions(dk, pair)
-        self.dd.append_model_predictions(pair, pred_df, do_preds, dk, len(dataframe))
+        self.dd.append_model_predictions(pair, pred_df, do_preds, dk, dataframe)
         dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
 
         return
 
     def check_if_feature_list_matches_strategy(
-        self, dataframe: DataFrame, dk: FreqaiDataKitchen
+        self, dk: FreqaiDataKitchen
     ) -> None:
         """
         Ensure user is passing the proper feature set if they are reusing an `identifier` pointing
@@ -429,11 +436,12 @@ class IFreqaiModel(ABC):
         :param dk: FreqaiDataKitchen = non-persistent data container/analyzer for
                    current coin/bot loop
         """
-        dk.find_features(dataframe)
+
         if "training_features_list_raw" in dk.data:
             feature_list = dk.data["training_features_list_raw"]
         else:
             feature_list = dk.data['training_features_list']
+
         if dk.training_features_list != feature_list:
             raise OperationalException(
                 "Trying to access pretrained model with `identifier` "
@@ -481,13 +489,16 @@ class IFreqaiModel(ABC):
         if self.freqai_info["feature_parameters"].get('noise_standard_deviation', 0):
             dk.add_noise_to_training_features()
 
-    def data_cleaning_predict(self, dk: FreqaiDataKitchen, dataframe: DataFrame) -> None:
+    def data_cleaning_predict(self, dk: FreqaiDataKitchen) -> None:
         """
         Base data cleaning method for predict.
         Functions here are complementary to the functions of data_cleaning_train.
         """
         ft_params = self.freqai_info["feature_parameters"]
 
+        # ensure user is feeding the correct indicators to the model
+        self.check_if_feature_list_matches_strategy(dk)
+
         if ft_params.get('inlier_metric_window', 0):
             dk.compute_inlier_metric(set_='predict')
 
@@ -505,9 +516,6 @@ class IFreqaiModel(ABC):
         if ft_params.get("use_DBSCAN_to_remove_outliers", False):
             dk.use_DBSCAN_to_remove_outliers(predict=True)
 
-        # ensure user is feeding the correct indicators to the model
-        self.check_if_feature_list_matches_strategy(dk.data_dictionary['prediction_features'], dk)
-
     def model_exists(self, dk: FreqaiDataKitchen) -> bool:
         """
         Given a pair and path, check if a model already exists
@@ -525,14 +533,13 @@ class IFreqaiModel(ABC):
         return file_exists
 
     def set_full_path(self) -> None:
+        """
+        Creates and sets the full path for the identifier
+        """
         self.full_path = Path(
-            self.config["user_data_dir"] / "models" / f"{self.freqai_info['identifier']}"
+            self.config["user_data_dir"] / "models" / f"{self.identifier}"
         )
         self.full_path.mkdir(parents=True, exist_ok=True)
-        shutil.copy(
-            self.config["config_files"][0],
-            Path(self.full_path, Path(self.config["config_files"][0]).name),
-        )
 
     def extract_data_and_train_model(
         self,
@@ -558,10 +565,9 @@ class IFreqaiModel(ABC):
             data_load_timerange, pair, dk
         )
 
-        with self.analysis_lock:
-            unfiltered_dataframe = dk.use_strategy_to_populate_indicators(
-                strategy, corr_dataframes, base_dataframes, pair
-            )
+        unfiltered_dataframe = dk.use_strategy_to_populate_indicators(
+            strategy, corr_dataframes, base_dataframes, pair
+        )
 
         unfiltered_dataframe = dk.slice_dataframe(new_trained_timerange, unfiltered_dataframe)
 
@@ -572,7 +578,7 @@ class IFreqaiModel(ABC):
         model = self.train(unfiltered_dataframe, pair, dk)
 
         self.dd.pair_dict[pair]["trained_timestamp"] = new_trained_timerange.stopts
-        dk.set_new_model_names(pair, new_trained_timerange)
+        dk.set_new_model_names(pair, new_trained_timerange.stopts)
         self.dd.save_data(model, pair, dk)
 
         if self.plot_features:
@@ -582,7 +588,7 @@ class IFreqaiModel(ABC):
             self.dd.purge_old_models()
 
     def set_initial_historic_predictions(
-        self, pred_df: DataFrame, dk: FreqaiDataKitchen, pair: str
+        self, pred_df: DataFrame, dk: FreqaiDataKitchen, pair: str, strat_df: DataFrame
     ) -> None:
         """
         This function is called only if the datadrawer failed to load an
@@ -601,11 +607,11 @@ class IFreqaiModel(ABC):
         If the user reuses an identifier on a subsequent instance,
         this function will not be called. In that case, "real" predictions
         will be appended to the loaded set of historic predictions.
-        :param: df: DataFrame = the dataframe containing the training feature data
-        :param: model: Any = A model which was `fit` using a common library such as
-        catboost or lightgbm
-        :param: dk: FreqaiDataKitchen = object containing methods for data analysis
-        :param: pair: str = current pair
+        :param df: DataFrame = the dataframe containing the training feature data
+        :param model: Any = A model which was `fit` using a common library such as
+                      catboost or lightgbm
+        :param dk: FreqaiDataKitchen = object containing methods for data analysis
+        :param pair: str = current pair
         """
 
         self.dd.historic_predictions[pair] = pred_df
@@ -623,7 +629,10 @@ class IFreqaiModel(ABC):
             hist_preds_df['DI_values'] = 0
 
         for return_str in dk.data['extra_returns_per_train']:
-            hist_preds_df[return_str] = 0
+            hist_preds_df[return_str] = dk.data['extra_returns_per_train'][return_str]
+
+        hist_preds_df['close_price'] = strat_df['close']
+        hist_preds_df['date_pred'] = strat_df['date']
 
         # # for keras type models, the conv_window needs to be prepended so
         # # viewing is correct in frequi
@@ -653,7 +662,7 @@ class IFreqaiModel(ABC):
 
         return
 
-    def inference_timer(self, do='start'):
+    def inference_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''):
         """
         Timer designed to track the cumulative time spent in FreqAI for one pass through
         the whitelist. This will check if the time spent is more than 1/4 the time
@@ -664,7 +673,10 @@ class IFreqaiModel(ABC):
             self.begin_time = time.time()
         elif do == 'stop':
             end = time.time()
-            self.inference_time += (end - self.begin_time)
+            time_spent = (end - self.begin_time)
+            if self.freqai_info.get('write_metrics_to_disk', False):
+                self.dd.update_metric_tracker('inference_time', time_spent, pair)
+            self.inference_time += time_spent
             if self.pair_it == self.total_pairs:
                 logger.info(
                     f'Total time spent inferencing pairlist {self.inference_time:.2f} seconds')
@@ -675,7 +687,7 @@ class IFreqaiModel(ABC):
                 self.inference_time = 0
         return
 
-    def train_timer(self, do='start'):
+    def train_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''):
         """
         Timer designed to track the cumulative time spent training the full pairlist in
         FreqAI.
@@ -685,7 +697,11 @@ class IFreqaiModel(ABC):
             self.begin_time_train = time.time()
         elif do == 'stop':
             end = time.time()
-            self.train_time += (end - self.begin_time_train)
+            time_spent = (end - self.begin_time_train)
+            if self.freqai_info.get('write_metrics_to_disk', False):
+                self.dd.collect_metrics(time_spent, pair)
+
+            self.train_time += time_spent
             if self.pair_it_train == self.total_pairs:
                 logger.info(
                     f'Total time spent training pairlist {self.train_time:.2f} seconds')
@@ -727,6 +743,74 @@ class IFreqaiModel(ABC):
                     f'Best approximation queue: {best_queue}')
         return best_queue
 
+    def cache_corr_pairlist_dfs(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> DataFrame:
+        """
+        Cache the corr_pairlist dfs to speed up performance for subsequent pairs during the
+        current candle.
+        :param dataframe: strategy fed dataframe
+        :param dk: datakitchen object for current asset
+        :return: dataframe to attach/extract cached corr_pair dfs to/from.
+        """
+
+        if self.get_corr_dataframes:
+            self.corr_dataframes = dk.extract_corr_pair_columns_from_populated_indicators(dataframe)
+            if not self.corr_dataframes:
+                logger.warning("Couldn't cache corr_pair dataframes for improved performance. "
+                               "Consider ensuring that the full coin/stake, e.g. XYZ/USD, "
+                               "is included in the column names when you are creating features "
+                               "in `populate_any_indicators()`.")
+            self.get_corr_dataframes = not bool(self.corr_dataframes)
+        elif self.corr_dataframes:
+            dataframe = dk.attach_corr_pair_columns(
+                dataframe, self.corr_dataframes, dk.pair)
+
+        return dataframe
+
+    def track_current_candle(self):
+        """
+        Checks if the latest candle appended by the datadrawer is
+        equivalent to the latest candle seen by FreqAI. If not, it
+        asks to refresh the cached corr_dfs, and resets the pair
+        counter.
+        """
+        if self.dd.current_candle > self.current_candle:
+            self.get_corr_dataframes = True
+            self.pair_it = 1
+            self.current_candle = self.dd.current_candle
+
+    def ensure_data_exists(self, dataframe_backtest: DataFrame,
+                           tr_backtest: TimeRange, pair: str) -> bool:
+        """
+        Check if the dataframe is empty, if not, report useful information to user.
+        :param dataframe_backtest: the backtesting dataframe, maybe empty.
+        :param tr_backtest: current backtesting timerange.
+        :param pair: current pair
+        :return: if the data exists or not
+        """
+        if self.config.get("freqai_backtest_live_models", False) and len(dataframe_backtest) == 0:
+            logger.info(f"No data found for pair {pair} from "
+                        f"from { tr_backtest.start_fmt} to {tr_backtest.stop_fmt}. "
+                        "Probably more than one training within the same candle period.")
+            return False
+        return True
+
+    def log_backtesting_progress(self, tr_train: TimeRange, pair: str,
+                                 train_it: int, total_trains: int):
+        """
+        Log the backtesting progress so user knows how many pairs have been trained and
+        how many more pairs/trains remain.
+        :param tr_train: the training timerange
+        :param train_it: the train iteration for the current pair (the sliding window progress)
+        :param pair: the current pair
+        :param total_trains: total trains (total number of slides for the sliding window)
+        """
+        if not self.config.get("freqai_backtest_live_models", False):
+            logger.info(
+                f"Training {pair}, {self.pair_it}/{self.total_pairs} pairs"
+                f" from {tr_train.start_fmt} "
+                f"to {tr_train.stop_fmt}, {train_it}/{total_trains} "
+                "trains"
+            )
     # Following methods which are overridden by user made prediction models.
     # See freqai/prediction_models/CatboostPredictionModel.py for an example.
 

freqtrade/freqai/prediction_models/CatboostClassifier.py

@@ -1,4 +1,6 @@
 import logging
+import sys
+from pathlib import Path
 from typing import Any, Dict
 
 from catboost import CatBoostClassifier, Pool
@@ -20,9 +22,8 @@ class CatboostClassifier(BaseClassifierModel):
     def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any:
         """
         User sets up the training and test data to fit their desired model here
-        :params:
-        :data_dictionary: the dictionary constructed by DataHandler to hold
-        all the training and test data/labels.
+        :param data_dictionary: the dictionary constructed by DataHandler to hold
+                                all the training and test data/labels.
         """
 
         train_data = Pool(
@@ -30,15 +31,25 @@ class CatboostClassifier(BaseClassifierModel):
             label=data_dictionary["train_labels"],
             weight=data_dictionary["train_weights"],
         )
+        if self.freqai_info.get("data_split_parameters", {}).get("test_size", 0.1) == 0:
+            test_data = None
+        else:
+            test_data = Pool(
+                data=data_dictionary["test_features"],
+                label=data_dictionary["test_labels"],
+                weight=data_dictionary["test_weights"],
+            )
 
         cbr = CatBoostClassifier(
-            allow_writing_files=False,
+            allow_writing_files=True,
             loss_function='MultiClass',
+            train_dir=Path(dk.data_path),
             **self.model_training_parameters,
         )
 
         init_model = self.get_init_model(dk.pair)
 
-        cbr.fit(train_data, init_model=init_model)
+        cbr.fit(X=train_data, eval_set=test_data, init_model=init_model,
+                log_cout=sys.stdout, log_cerr=sys.stderr)
 
         return cbr

freqtrade/freqai/prediction_models/CatboostClassifierMultiTarget.py

@@ -0,0 +1,74 @@
+import logging
+import sys
+from pathlib import Path
+from typing import Any, Dict
+
+from catboost import CatBoostClassifier, Pool
+
+from freqtrade.freqai.base_models.BaseClassifierModel import BaseClassifierModel
+from freqtrade.freqai.base_models.FreqaiMultiOutputClassifier import FreqaiMultiOutputClassifier
+from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
+
+
+logger = logging.getLogger(__name__)
+
+
+class CatboostClassifierMultiTarget(BaseClassifierModel):
+    """
+    User created prediction model. The class needs to override three necessary
+    functions, predict(), train(), fit(). The class inherits ModelHandler which
+    has its own DataHandler where data is held, saved, loaded, and managed.
+    """
+
+    def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any:
+        """
+        User sets up the training and test data to fit their desired model here
+        :param data_dictionary: the dictionary constructed by DataHandler to hold
+                                all the training and test data/labels.
+        """
+
+        cbc = CatBoostClassifier(
+            allow_writing_files=True,
+            loss_function='MultiClass',
+            train_dir=Path(dk.data_path),
+            **self.model_training_parameters,
+        )
+
+        X = data_dictionary["train_features"]
+        y = data_dictionary["train_labels"]
+
+        sample_weight = data_dictionary["train_weights"]
+
+        eval_sets = [None] * y.shape[1]
+
+        if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) != 0:
+            eval_sets = [None] * data_dictionary['test_labels'].shape[1]
+
+            for i in range(data_dictionary['test_labels'].shape[1]):
+                eval_sets[i] = Pool(
+                    data=data_dictionary["test_features"],
+                    label=data_dictionary["test_labels"].iloc[:, i],
+                    weight=data_dictionary["test_weights"],
+                )
+
+        init_model = self.get_init_model(dk.pair)
+
+        if init_model:
+            init_models = init_model.estimators_
+        else:
+            init_models = [None] * y.shape[1]
+
+        fit_params = []
+        for i in range(len(eval_sets)):
+            fit_params.append({
+                'eval_set': eval_sets[i], 'init_model': init_models[i],
+                'log_cout': sys.stdout, 'log_cerr': sys.stderr,
+            })
+
+        model = FreqaiMultiOutputClassifier(estimator=cbc)
+        thread_training = self.freqai_info.get('multitarget_parallel_training', False)
+        if thread_training:
+            model.n_jobs = y.shape[1]
+        model.fit(X=X, y=y, sample_weight=sample_weight, fit_params=fit_params)
+
+        return model

freqtrade/freqai/prediction_models/CatboostRegressor.py

@@ -1,4 +1,6 @@
 import logging
+import sys
+from pathlib import Path
 from typing import Any, Dict
 
 from catboost import CatBoostRegressor, Pool
@@ -41,10 +43,12 @@ class CatboostRegressor(BaseRegressionModel):
         init_model = self.get_init_model(dk.pair)
 
         model = CatBoostRegressor(
-            allow_writing_files=False,
+            allow_writing_files=True,
+            train_dir=Path(dk.data_path),
             **self.model_training_parameters,
         )
 
-        model.fit(X=train_data, eval_set=test_data, init_model=init_model)
+        model.fit(X=train_data, eval_set=test_data, init_model=init_model,
+                  log_cout=sys.stdout, log_cerr=sys.stderr)
 
         return model

freqtrade/freqai/prediction_models/CatboostRegressorMultiTarget.py

@@ -1,4 +1,6 @@
 import logging
+import sys
+from pathlib import Path
 from typing import Any, Dict
 
 from catboost import CatBoostRegressor, Pool
@@ -26,7 +28,8 @@ class CatboostRegressorMultiTarget(BaseRegressionModel):
         """
 
         cbr = CatBoostRegressor(
-            allow_writing_files=False,
+            allow_writing_files=True,
+            train_dir=Path(dk.data_path),
             **self.model_training_parameters,
         )
 
@@ -56,8 +59,10 @@ class CatboostRegressorMultiTarget(BaseRegressionModel):
 
         fit_params = []
         for i in range(len(eval_sets)):
-            fit_params.append(
-                {'eval_set': eval_sets[i],  'init_model': init_models[i]})
+            fit_params.append({
+                    'eval_set': eval_sets[i],  'init_model': init_models[i],
+                    'log_cout': sys.stdout, 'log_cerr': sys.stderr,
+                 })
 
         model = FreqaiMultiOutputRegressor(estimator=cbr)
         thread_training = self.freqai_info.get('multitarget_parallel_training', False)

freqtrade/freqai/prediction_models/LightGBMClassifier.py

@@ -20,9 +20,8 @@ class LightGBMClassifier(BaseClassifierModel):
     def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any:
         """
         User sets up the training and test data to fit their desired model here
-        :params:
-        :data_dictionary: the dictionary constructed by DataHandler to hold
-        all the training and test data/labels.
+        :param data_dictionary: the dictionary constructed by DataHandler to hold
+                                all the training and test data/labels.
         """
 
         if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) == 0:

freqtrade/freqai/prediction_models/LightGBMClassifierMultiTarget.py

@@ -0,0 +1,64 @@
+import logging
+from typing import Any, Dict
+
+from lightgbm import LGBMClassifier
+
+from freqtrade.freqai.base_models.BaseClassifierModel import BaseClassifierModel
+from freqtrade.freqai.base_models.FreqaiMultiOutputClassifier import FreqaiMultiOutputClassifier
+from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
+
+
+logger = logging.getLogger(__name__)
+
+
+class LightGBMClassifierMultiTarget(BaseClassifierModel):
+    """
+    User created prediction model. The class needs to override three necessary
+    functions, predict(), train(), fit(). The class inherits ModelHandler which
+    has its own DataHandler where data is held, saved, loaded, and managed.
+    """
+
+    def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any:
+        """
+        User sets up the training and test data to fit their desired model here
+        :param data_dictionary: the dictionary constructed by DataHandler to hold
+                                all the training and test data/labels.
+        """
+
+        lgb = LGBMClassifier(**self.model_training_parameters)
+
+        X = data_dictionary["train_features"]
+        y = data_dictionary["train_labels"]
+        sample_weight = data_dictionary["train_weights"]
+
+        eval_weights = None
+        eval_sets = [None] * y.shape[1]
+
+        if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) != 0:
+            eval_weights = [data_dictionary["test_weights"]]
+            eval_sets = [(None, None)] * data_dictionary['test_labels'].shape[1]  # type: ignore
+            for i in range(data_dictionary['test_labels'].shape[1]):
+                eval_sets[i] = (  # type: ignore
+                    data_dictionary["test_features"],
+                    data_dictionary["test_labels"].iloc[:, i]
+                )
+
+        init_model = self.get_init_model(dk.pair)
+        if init_model:
+            init_models = init_model.estimators_
+        else:
+            init_models = [None] * y.shape[1]
+
+        fit_params = []
+        for i in range(len(eval_sets)):
+            fit_params.append(
+                {'eval_set': eval_sets[i], 'eval_sample_weight': eval_weights,
+                 'init_model': init_models[i]})
+
+        model = FreqaiMultiOutputClassifier(estimator=lgb)
+        thread_training = self.freqai_info.get('multitarget_parallel_training', False)
+        if thread_training:
+            model.n_jobs = y.shape[1]
+        model.fit(X=X, y=y, sample_weight=sample_weight, fit_params=fit_params)
+
+        return model

freqtrade/freqai/prediction_models/XGBoostClassifier.py

@@ -26,9 +26,8 @@ class XGBoostClassifier(BaseClassifierModel):
     def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any:
         """
         User sets up the training and test data to fit their desired model here
-        :params:
-        :data_dictionary: the dictionary constructed by DataHandler to hold
-        all the training and test data/labels.
+        :param data_dictionary: the dictionary constructed by DataHandler to hold
+                                all the training and test data/labels.
         """
 
         X = data_dictionary["train_features"].to_numpy()
@@ -65,7 +64,7 @@ class XGBoostClassifier(BaseClassifierModel):
     ) -> Tuple[DataFrame, npt.NDArray[np.int_]]:
         """
         Filter the prediction features data and predict with it.
-        :param: unfiltered_df: Full dataframe for the current backtest period.
+        :param unfiltered_df: Full dataframe for the current backtest period.
         :return:
         :pred_df: dataframe containing the predictions
         :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove

freqtrade/freqai/prediction_models/XGBoostRFClassifier.py

@@ -0,0 +1,84 @@
+import logging
+from typing import Any, Dict, Tuple
+
+import numpy as np
+import numpy.typing as npt
+import pandas as pd
+from pandas import DataFrame
+from pandas.api.types import is_integer_dtype
+from sklearn.preprocessing import LabelEncoder
+from xgboost import XGBRFClassifier
+
+from freqtrade.freqai.base_models.BaseClassifierModel import BaseClassifierModel
+from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
+
+
+logger = logging.getLogger(__name__)
+
+
+class XGBoostRFClassifier(BaseClassifierModel):
+    """
+    User created prediction model. The class needs to override three necessary
+    functions, predict(), train(), fit(). The class inherits ModelHandler which
+    has its own DataHandler where data is held, saved, loaded, and managed.
+    """
+
+    def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any:
+        """
+        User sets up the training and test data to fit their desired model here
+        :param data_dictionary: the dictionary constructed by DataHandler to hold
+            all the training and test data/labels.
+        """
+
+        X = data_dictionary["train_features"].to_numpy()
+        y = data_dictionary["train_labels"].to_numpy()[:, 0]
+
+        le = LabelEncoder()
+        if not is_integer_dtype(y):
+            y = pd.Series(le.fit_transform(y), dtype="int64")
+
+        if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) == 0:
+            eval_set = None
+        else:
+            test_features = data_dictionary["test_features"].to_numpy()
+            test_labels = data_dictionary["test_labels"].to_numpy()[:, 0]
+
+            if not is_integer_dtype(test_labels):
+                test_labels = pd.Series(le.transform(test_labels), dtype="int64")
+
+            eval_set = [(test_features, test_labels)]
+
+        train_weights = data_dictionary["train_weights"]
+
+        init_model = self.get_init_model(dk.pair)
+
+        model = XGBRFClassifier(**self.model_training_parameters)
+
+        model.fit(X=X, y=y, eval_set=eval_set, sample_weight=train_weights,
+                  xgb_model=init_model)
+
+        return model
+
+    def predict(
+        self, unfiltered_df: DataFrame, dk: FreqaiDataKitchen, **kwargs
+    ) -> Tuple[DataFrame, npt.NDArray[np.int_]]:
+        """
+        Filter the prediction features data and predict with it.
+        :param  unfiltered_df: Full dataframe for the current backtest period.
+        :return:
+        :pred_df: dataframe containing the predictions
+        :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove
+        data (NaNs) or felt uncertain about data (PCA and DI index)
+        """
+
+        (pred_df, dk.do_predict) = super().predict(unfiltered_df, dk, **kwargs)
+
+        le = LabelEncoder()
+        label = dk.label_list[0]
+        labels_before = list(dk.data['labels_std'].keys())
+        labels_after = le.fit_transform(labels_before).tolist()
+        pred_df[label] = le.inverse_transform(pred_df[label])
+        pred_df = pred_df.rename(
+            columns={labels_after[i]: labels_before[i] for i in range(len(labels_before))})
+
+        return (pred_df, dk.do_predict)

freqtrade/freqai/prediction_models/XGBoostRFRegressor.py

@@ -0,0 +1,46 @@
+import logging
+from typing import Any, Dict
+
+from xgboost import XGBRFRegressor
+
+from freqtrade.freqai.base_models.BaseRegressionModel import BaseRegressionModel
+from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
+
+
+logger = logging.getLogger(__name__)
+
+
+class XGBoostRFRegressor(BaseRegressionModel):
+    """
+    User created prediction model. The class needs to override three necessary
+    functions, predict(), train(), fit(). The class inherits ModelHandler which
+    has its own DataHandler where data is held, saved, loaded, and managed.
+    """
+
+    def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any:
+        """
+        User sets up the training and test data to fit their desired model here
+        :param data_dictionary: the dictionary constructed by DataHandler to hold
+                                all the training and test data/labels.
+        """
+
+        X = data_dictionary["train_features"]
+        y = data_dictionary["train_labels"]
+
+        if self.freqai_info.get("data_split_parameters", {}).get("test_size", 0.1) == 0:
+            eval_set = None
+            eval_weights = None
+        else:
+            eval_set = [(data_dictionary["test_features"], data_dictionary["test_labels"])]
+            eval_weights = [data_dictionary['test_weights']]
+
+        sample_weight = data_dictionary["train_weights"]
+
+        xgb_model = self.get_init_model(dk.pair)
+
+        model = XGBRFRegressor(**self.model_training_parameters)
+
+        model.fit(X=X, y=y, sample_weight=sample_weight, eval_set=eval_set,
+                  sample_weight_eval_set=eval_weights, xgb_model=xgb_model)
+
+        return model

freqtrade/freqai/prediction_models/XGBoostRegressor.py

@@ -29,6 +29,7 @@ class XGBoostRegressor(BaseRegressionModel):
 
         if self.freqai_info.get("data_split_parameters", {}).get("test_size", 0.1) == 0:
             eval_set = None
+            eval_weights = None
         else:
             eval_set = [(data_dictionary["test_features"], data_dictionary["test_labels"])]
             eval_weights = [data_dictionary['test_weights']]

freqtrade/freqai/utils.py

@@ -1,9 +1,11 @@
 import logging
 from datetime import datetime, timezone
-from typing import Any
+from pathlib import Path
+from typing import Any, Dict
 
 import numpy as np
 import pandas as pd
+import rapidjson
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import Config
@@ -191,3 +193,41 @@ def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen,
         fig.update_layout(title_text=f"Best and worst features by importance {pair}")
         label = label.replace('&', '').replace('%', '')  # escape two FreqAI specific characters
         store_plot_file(fig, f"{dk.model_filename}-{label}.html", dk.data_path)
+
+
+def record_params(config: Dict[str, Any], full_path: Path) -> None:
+    """
+    Records run params in the full path for reproducibility
+    """
+    params_record_path = full_path / "run_params.json"
+
+    run_params = {
+        "freqai": config.get('freqai', {}),
+        "timeframe": config.get('timeframe'),
+        "stake_amount": config.get('stake_amount'),
+        "stake_currency": config.get('stake_currency'),
+        "max_open_trades": config.get('max_open_trades'),
+        "pairs": config.get('exchange', {}).get('pair_whitelist')
+    }
+
+    with open(params_record_path, "w") as handle:
+        rapidjson.dump(
+            run_params,
+            handle,
+            indent=4,
+            default=str,
+            number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN
+        )
+
+
+def get_timerange_backtest_live_models(config: Config) -> str:
+    """
+    Returns a formated timerange for backtest live/ready models
+    :param config: Configuration dictionary
+
+    :return: a string timerange (format example: '20220801-20220822')
+    """
+    dk = FreqaiDataKitchen(config)
+    models_path = dk.get_full_models_path(config)
+    timerange, _ = dk.get_timerange_and_assets_end_dates_from_ready_models(models_path)
+    return timerange.timerange_str

freqtrade/freqtradebot.py

@@ -82,7 +82,10 @@ class FreqtradeBot(LoggingMixin):
         # Keep this at the end of this initialization method.
         self.rpc: RPCManager = RPCManager(self)
 
-        self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists, self.rpc)
+        self.dataprovider = DataProvider(self.config, self.exchange, rpc=self.rpc)
+        self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider)
+
+        self.dataprovider.add_pairlisthandler(self.pairlists)
 
         # Attach Dataprovider to strategy instance
         self.strategy.dp = self.dataprovider
@@ -188,10 +191,10 @@ class FreqtradeBot(LoggingMixin):
         # Check whether markets have to be reloaded and reload them when it's needed
         self.exchange.reload_markets()
 
-        self.update_closed_trades_without_assigned_fees()
+        self.update_trades_without_assigned_fees()
 
         # Query trades from persistence layer
-        trades = Trade.get_open_trades()
+        trades: List[Trade] = Trade.get_open_trades()
 
         self.active_pair_whitelist = self._refresh_active_whitelist(trades)
 
@@ -351,7 +354,7 @@ class FreqtradeBot(LoggingMixin):
         if self.trading_mode == TradingMode.FUTURES:
             self._schedule.run_pending()
 
-    def update_closed_trades_without_assigned_fees(self):
+    def update_trades_without_assigned_fees(self) -> None:
         """
         Update closed trades without close fees assigned.
         Only acts when Orders are in the database, otherwise the last order-id is unknown.
@@ -376,17 +379,18 @@ class FreqtradeBot(LoggingMixin):
                                             stoploss_order=order.ft_order_side == 'stoploss',
                                             send_msg=False)
 
-        trades: List[Trade] = Trade.get_open_trades_without_assigned_fees()
+        trades = Trade.get_open_trades_without_assigned_fees()
         for trade in trades:
-            if trade.is_open and not trade.fee_updated(trade.entry_side):
-                order = trade.select_order(trade.entry_side, False)
-                open_order = trade.select_order(trade.entry_side, True)
-                if order and open_order is None:
-                    logger.info(
-                        f"Updating {trade.entry_side}-fee on trade {trade}"
-                        f"for order {order.order_id}."
-                    )
-                    self.update_trade_state(trade, order.order_id, send_msg=False)
+            with self._exit_lock:
+                if trade.is_open and not trade.fee_updated(trade.entry_side):
+                    order = trade.select_order(trade.entry_side, False)
+                    open_order = trade.select_order(trade.entry_side, True)
+                    if order and open_order is None:
+                        logger.info(
+                            f"Updating {trade.entry_side}-fee on trade {trade}"
+                            f"for order {order.order_id}."
+                        )
+                        self.update_trade_state(trade, order.order_id, send_msg=False)
 
     def handle_insufficient_funds(self, trade: Trade):
         """
@@ -597,7 +601,7 @@ class FreqtradeBot(LoggingMixin):
             # We should decrease our position
             amount = self.exchange.amount_to_contract_precision(
                 trade.pair,
-                abs(float(FtPrecise(stake_amount) / FtPrecise(current_exit_rate))))
+                abs(float(FtPrecise(stake_amount * trade.leverage) / FtPrecise(current_exit_rate))))
             if amount > trade.amount:
                 # This is currently ineffective as remaining would become < min tradable
                 # Fixing this would require checking for 0.0 there -
@@ -823,6 +827,8 @@ class FreqtradeBot(LoggingMixin):
                 co = self.exchange.cancel_stoploss_order_with_result(
                     trade.stoploss_order_id, trade.pair, trade.amount)
                 trade.update_order(co)
+                # Reset stoploss order id.
+                trade.stoploss_order_id = None
             except InvalidOrderException:
                 logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id}")
         return trade
@@ -979,7 +985,7 @@ class FreqtradeBot(LoggingMixin):
 # SELL / exit positions / close trades logic and methods
 #
 
-    def exit_positions(self, trades: List[Any]) -> int:
+    def exit_positions(self, trades: List[Trade]) -> int:
         """
         Tries to execute exit orders for open trades (positions)
         """
@@ -1007,7 +1013,7 @@ class FreqtradeBot(LoggingMixin):
 
     def handle_trade(self, trade: Trade) -> bool:
         """
-        Sells/exits_short the current pair if the threshold is reached and updates the trade record.
+        Exits the current pair if the threshold is reached and updates the trade record.
         :return: True if trade has been sold/exited_short, False otherwise
         """
         if not trade.is_open:
@@ -1130,10 +1136,8 @@ class FreqtradeBot(LoggingMixin):
             trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
             self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
                                     stoploss_order=True)
-            # Lock pair for one candle to prevent immediate rebuys
-            self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
-                                    reason='Auto lock')
             self._notify_exit(trade, "stoploss", True)
+            self.handle_protections(trade.pair, trade.trade_direction)
             return True
 
         if trade.open_order_id or not trade.is_open:
@@ -1166,7 +1170,6 @@ class FreqtradeBot(LoggingMixin):
             if self.create_stoploss_order(trade=trade, stop_price=trade.stoploss_or_liquidation):
                 return False
             else:
-                trade.stoploss_order_id = None
                 logger.warning('Stoploss order was cancelled, but unable to recreate one.')
 
         # Finally we check if stoploss on exchange should be moved up because of trailing.
@@ -1308,7 +1311,7 @@ class FreqtradeBot(LoggingMixin):
                     # place new order only if new price is supplied
                     self.execute_entry(
                         pair=trade.pair,
-                        stake_amount=(order_obj.remaining * order_obj.price),
+                        stake_amount=(order_obj.remaining * order_obj.price / trade.leverage),
                         price=adjusted_entry_price,
                         trade=trade,
                         is_short=trade.is_short,
@@ -1340,11 +1343,12 @@ class FreqtradeBot(LoggingMixin):
             replacing: Optional[bool] = False
     ) -> bool:
         """
-        Buy cancel - cancel order
+        entry cancel - cancel order
         :param replacing: Replacing order - prevent trade deletion.
-        :return: True if order was fully cancelled
+        :return: True if trade was fully cancelled
         """
         was_trade_fully_canceled = False
+        side = trade.entry_side.capitalize()
 
         # Cancelled orders may have the status of 'canceled' or 'closed'
         if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
@@ -1371,7 +1375,6 @@ class FreqtradeBot(LoggingMixin):
             corder = order
             reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
 
-        side = trade.entry_side.capitalize()
         logger.info('%s order %s for %s.', side, reason, trade)
 
         # Using filled to determine the filled amount
@@ -1385,24 +1388,15 @@ class FreqtradeBot(LoggingMixin):
                 was_trade_fully_canceled = True
                 reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}"
             else:
-                # FIXME TODO: This could possibly reworked to not duplicate the code 15 lines below.
                 self.update_trade_state(trade, trade.open_order_id, corder)
                 trade.open_order_id = None
                 logger.info(f'{side} Order timeout for {trade}.')
         else:
-            # if trade is partially complete, edit the stake details for the trade
-            # and close the order
-            # cancel_order may not contain the full order dict, so we need to fallback
-            # to the order dict acquired before cancelling.
-            # we need to fall back to the values from order if corder does not contain these keys.
-            trade.amount = filled_amount
-            # * Check edge cases, we don't want to make leverage > 1.0 if we don't have to
-            # * (for leverage modes which aren't isolated futures)
-
-            trade.stake_amount = trade.amount * trade.open_rate / trade.leverage
+            # update_trade_state (and subsequently recalc_trade_from_orders) will handle updates
+            # to the trade object
             self.update_trade_state(trade, trade.open_order_id, corder)
-
             trade.open_order_id = None
+
             logger.info(f'Partial {trade.entry_side} order timeout for {trade}.')
             reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}"
 
@@ -1417,58 +1411,73 @@ class FreqtradeBot(LoggingMixin):
         :return: True if exit order was cancelled, false otherwise
         """
         cancelled = False
-        # if trade is not partially completed, just cancel the order
-        if order['remaining'] == order['amount'] or order.get('filled') == 0.0:
-            if not self.exchange.check_order_canceled_empty(order):
-                try:
-                    # if trade is not partially completed, just delete the order
-                    co = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
-                                                                trade.amount)
-                    trade.update_order(co)
-                except InvalidOrderException:
-                    logger.exception(
-                        f"Could not cancel {trade.exit_side} order {trade.open_order_id}")
-                    return False
-                logger.info('%s order %s for %s.', trade.exit_side.capitalize(), reason, trade)
-            else:
-                reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
-                logger.info('%s order %s for %s.', trade.exit_side.capitalize(), reason, trade)
-                trade.update_order(order)
+        # Cancelled orders may have the status of 'canceled' or 'closed'
+        if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
+            filled_val: float = order.get('filled', 0.0) or 0.0
+            filled_rem_stake = trade.stake_amount - filled_val * trade.open_rate
+            minstake = self.exchange.get_min_pair_stake_amount(
+                trade.pair, trade.open_rate, self.strategy.stoploss)
+            # Double-check remaining amount
+            if filled_val > 0:
+                reason = constants.CANCEL_REASON['PARTIALLY_FILLED']
+                if minstake and filled_rem_stake < minstake:
+                    logger.warning(
+                        f"Order {trade.open_order_id} for {trade.pair} not cancelled, as "
+                        f"the filled amount of {filled_val} would result in an unexitable trade.")
+                    reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
 
+                    self._notify_exit_cancel(
+                        trade,
+                        order_type=self.strategy.order_types['exit'],
+                        reason=reason, order_id=order['id'],
+                        sub_trade=trade.amount != order['amount']
+                    )
+                    return False
+
+            try:
+                co = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
+                                                            trade.amount)
+            except InvalidOrderException:
+                logger.exception(
+                    f"Could not cancel {trade.exit_side} order {trade.open_order_id}")
+                return False
             trade.close_rate = None
             trade.close_rate_requested = None
             trade.close_profit = None
             trade.close_profit_abs = None
-            trade.close_date = None
-            trade.is_open = True
-            trade.open_order_id = None
-            trade.exit_reason = None
+            # Set exit_reason for fill message
+            exit_reason_prev = trade.exit_reason
+            trade.exit_reason = trade.exit_reason + f", {reason}" if trade.exit_reason else reason
+            self.update_trade_state(trade, trade.open_order_id, co)
+            # Order might be filled above in odd timing issues.
+            if co.get('status') in ('canceled', 'cancelled'):
+                trade.exit_reason = None
+                trade.open_order_id = None
+            else:
+                trade.exit_reason = exit_reason_prev
+
+            logger.info(f'{trade.exit_side.capitalize()} order {reason} for {trade}.')
             cancelled = True
-            self.wallets.update()
         else:
-            # TODO: figure out how to handle partially complete sell orders
-            reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
-            cancelled = False
+            reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
+            logger.info(f'{trade.exit_side.capitalize()} order {reason} for {trade}.')
+            self.update_trade_state(trade, trade.open_order_id, order)
+            trade.open_order_id = None
 
-        order_obj = trade.select_order_by_order_id(order['id'])
-        if not order_obj:
-            raise DependencyException(
-                f"Order_obj not found for {order['id']}. This should not have happened.")
-
-        sub_trade = order_obj.amount != trade.amount
         self._notify_exit_cancel(
             trade,
             order_type=self.strategy.order_types['exit'],
-            reason=reason, order=order_obj, sub_trade=sub_trade
+            reason=reason, order_id=order['id'], sub_trade=trade.amount != order['amount']
         )
         return cancelled
 
-    def _safe_exit_amount(self, pair: str, amount: float) -> float:
+    def _safe_exit_amount(self, trade: Trade, pair: str, amount: float) -> float:
         """
         Get sellable amount.
         Should be trade.amount - but will fall back to the available amount if necessary.
         This should cover cases where get_real_amount() was not able to update the amount
         for whatever reason.
+        :param trade: Trade we're working with
         :param pair: Pair we're trying to sell
         :param amount: amount we expect to be available
         :return: amount to sell
@@ -1487,6 +1496,7 @@ class FreqtradeBot(LoggingMixin):
             return amount
         elif wallet_amount > amount * 0.98:
             logger.info(f"{pair} - Falling back to wallet-amount {wallet_amount} -> {amount}.")
+            trade.amount = wallet_amount
             return wallet_amount
         else:
             raise DependencyException(
@@ -1545,7 +1555,7 @@ class FreqtradeBot(LoggingMixin):
             # Emergency sells (default to market!)
             order_type = self.strategy.order_types.get("emergency_exit", "market")
 
-        amount = self._safe_exit_amount(trade.pair, sub_trade_amt or trade.amount)
+        amount = self._safe_exit_amount(trade, trade.pair, sub_trade_amt or trade.amount)
         time_in_force = self.strategy.order_time_in_force['exit']
 
         if (exit_check.exit_type != ExitType.LIQUIDATION
@@ -1585,11 +1595,6 @@ class FreqtradeBot(LoggingMixin):
         trade.close_rate_requested = limit
         trade.exit_reason = exit_reason
 
-        if not sub_trade_amt:
-            # Lock pair for one candle to prevent immediate re-trading
-            self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
-                                    reason='Auto lock')
-
         self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj)
         # In case of market sell orders the order can be closed immediately
         if order.get('status', 'unknown') in ('closed', 'expired'):
@@ -1656,7 +1661,7 @@ class FreqtradeBot(LoggingMixin):
         self.rpc.send_msg(msg)
 
     def _notify_exit_cancel(self, trade: Trade, order_type: str, reason: str,
-                            order: Order, sub_trade: bool = False) -> None:
+                            order_id: str, sub_trade: bool = False) -> None:
         """
         Sends rpc notification when a sell cancel occurred.
         """
@@ -1665,6 +1670,11 @@ class FreqtradeBot(LoggingMixin):
         else:
             trade.exit_order_status = reason
 
+        order = trade.select_order_by_order_id(order_id)
+        if not order:
+            raise DependencyException(
+                f"Order_obj not found for {order_id}. This should not have happened.")
+
         profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
         profit_trade = trade.calc_profit(rate=profit_rate)
         current_rate = self.exchange.get_rate(
@@ -1700,11 +1710,6 @@ class FreqtradeBot(LoggingMixin):
             'stake_amount': trade.stake_amount,
         }
 
-        if 'fiat_display_currency' in self.config:
-            msg.update({
-                'fiat_currency': self.config['fiat_display_currency'],
-            })
-
         # Send the message
         self.rpc.send_msg(msg)
 
@@ -1799,6 +1804,8 @@ class FreqtradeBot(LoggingMixin):
             self._notify_enter(trade, order, fill=True, sub_trade=sub_trade)
 
     def handle_protections(self, pair: str, side: LongShort) -> None:
+        # Lock pair for one candle to prevent immediate rebuys
+        self.strategy.lock_pair(pair, datetime.now(timezone.utc), reason='Auto lock')
         prot_trig = self.protections.stop_per_pair(pair, side=side)
         if prot_trig:
             msg = {'type': RPCMessageType.PROTECTION_TRIGGER, }
@@ -1820,7 +1827,7 @@ class FreqtradeBot(LoggingMixin):
         never in base currency.
         """
         self.wallets.update()
-        amount_ = amount
+        amount_ = trade.amount
         if order_obj.ft_order_side == trade.exit_side or order_obj.ft_order_side == 'stoploss':
             # check against remaining amount!
             amount_ = trade.amount - amount

freqtrade/leverage/interest.py

@@ -35,9 +35,5 @@ def interest(
     elif exchange_name == "kraken":
         # Rounded based on https://kraken-fees-calculator.github.io/
         return borrowed * rate * (one + FtPrecise(ceil(hours / four)))
-    elif exchange_name == "ftx":
-        # As Explained under #Interest rates section in
-        # https://help.ftx.com/hc/en-us/articles/360053007671-Spot-Margin-Trading-Explainer
-        return borrowed * rate * FtPrecise(ceil(hours)) / twenty_four
     else:
         raise OperationalException(f"Leverage not available on {exchange_name} with freqtrade")

freqtrade/misc.py

@@ -6,11 +6,12 @@ import logging
 import re
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Iterator, List
+from typing import Any, Dict, Iterator, List, Mapping, Union
 from typing.io import IO
 from urllib.parse import urlparse
 
-import pandas
+import orjson
+import pandas as pd
 import rapidjson
 
 from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN
@@ -186,7 +187,10 @@ def safe_value_fallback(obj: dict, key1: str, key2: str, default_value=None):
     return default_value
 
 
-def safe_value_fallback2(dict1: dict, dict2: dict, key1: str, key2: str, default_value=None):
+dictMap = Union[Dict[str, Any], Mapping[str, Any]]
+
+
+def safe_value_fallback2(dict1: dictMap, dict2: dictMap, key1: str, key2: str, default_value=None):
     """
     Search a value in dict1, return this if it's not None.
     Fall back to dict2 - return key2 from dict2 if it's not None.
@@ -253,29 +257,37 @@ def parse_db_uri_for_logging(uri: str):
     return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')
 
 
-def dataframe_to_json(dataframe: pandas.DataFrame) -> str:
+def dataframe_to_json(dataframe: pd.DataFrame) -> str:
     """
     Serialize a DataFrame for transmission over the wire using JSON
     :param dataframe: A pandas DataFrame
     :returns: A JSON string of the pandas DataFrame
     """
-    return dataframe.to_json(orient='split')
+    # https://github.com/pandas-dev/pandas/issues/24889
+    # https://github.com/pandas-dev/pandas/issues/40443
+    # We need to convert to a dict to avoid mem leak
+    def default(z):
+        if isinstance(z, pd.Timestamp):
+            return z.timestamp() * 1e3
+        raise TypeError
+
+    return str(orjson.dumps(dataframe.to_dict(orient='split'), default=default), 'utf-8')
 
 
-def json_to_dataframe(data: str) -> pandas.DataFrame:
+def json_to_dataframe(data: str) -> pd.DataFrame:
     """
     Deserialize JSON into a DataFrame
     :param data: A JSON string
     :returns: A pandas DataFrame from the JSON string
     """
-    dataframe = pandas.read_json(data, orient='split')
+    dataframe = pd.read_json(data, orient='split')
     if 'date' in dataframe.columns:
-        dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)
+        dataframe['date'] = pd.to_datetime(dataframe['date'], unit='ms', utc=True)
 
     return dataframe
 
 
-def remove_entry_exit_signals(dataframe: pandas.DataFrame):
+def remove_entry_exit_signals(dataframe: pd.DataFrame):
     """
     Remove Entry and Exit signals from a DataFrame
 

freqtrade/optimize/backtesting.py

@@ -110,7 +110,7 @@ class Backtesting:
         self.timeframe = str(self.config.get('timeframe'))
         self.timeframe_min = timeframe_to_minutes(self.timeframe)
         self.init_backtest_detail()
-        self.pairlists = PairListManager(self.exchange, self.config)
+        self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider)
         if 'VolumePairList' in self.pairlists.name_list:
             raise OperationalException("VolumePairList not allowed for backtesting. "
                                        "Please use StaticPairList instead.")
@@ -134,6 +134,10 @@ class Backtesting:
             self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0])
         self.precision_mode = self.exchange.precisionMode
 
+        if self.config.get('freqai_backtest_live_models', False):
+            from freqtrade.freqai.utils import get_timerange_backtest_live_models
+            self.config['timerange'] = get_timerange_backtest_live_models(self.config)
+
         self.timerange = TimeRange.parse_timerange(
             None if self.config.get('timerange') is None else str(self.config.get('timerange')))
 
@@ -151,6 +155,8 @@ class Backtesting:
         self.trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT)
         # strategies which define "can_short=True" will fail to load in Spot mode.
         self._can_short = self.trading_mode != TradingMode.SPOT
+        self._position_stacking: bool = self.config.get('position_stacking', False)
+        self.enable_protections: bool = self.config.get('enable_protections', False)
 
         self.init_backtest()
 
@@ -160,7 +166,7 @@ class Backtesting:
         PairLocks.use_db = True
         Trade.use_db = True
 
-    def init_backtest_detail(self):
+    def init_backtest_detail(self) -> None:
         # Load detail timeframe if specified
         self.timeframe_detail = str(self.config.get('timeframe_detail', ''))
         if self.timeframe_detail:
@@ -540,7 +546,7 @@ class Backtesting:
 
         if stake_amount is not None and stake_amount < 0.0:
             amount = amount_to_contract_precision(
-                abs(stake_amount) / current_rate, trade.amount_precision,
+                abs(stake_amount * trade.leverage) / current_rate, trade.amount_precision,
                 self.precision_mode, trade.contract_size)
             if amount == 0.0:
                 return trade
@@ -617,13 +623,16 @@ class Backtesting:
                     exit_reason = row[EXIT_TAG_IDX]
                 # Custom exit pricing only for exit-signals
                 if order_type == 'limit':
-                    close_rate = strategy_safe_wrapper(self.strategy.custom_exit_price,
-                                                       default_retval=close_rate)(
+                    rate = strategy_safe_wrapper(self.strategy.custom_exit_price,
+                                                 default_retval=close_rate)(
                         pair=trade.pair,
                         trade=trade,  # type: ignore[arg-type]
                         current_time=exit_candle_time,
                         proposed_rate=close_rate, current_profit=current_profit,
                         exit_tag=exit_reason)
+                    if rate != close_rate:
+                        close_rate = price_to_precision(rate, trade.price_precision,
+                                                        self.precision_mode)
                     # We can't place orders lower than current low.
                     # freqtrade does not support this in live, and the order would fill immediately
                     if trade.is_short:
@@ -660,7 +669,6 @@ class Backtesting:
         # amount = amount or trade.amount
         amount = amount_to_contract_precision(amount or trade.amount, trade.amount_precision,
                                               self.precision_mode, trade.contract_size)
-        rate = price_to_precision(close_rate, trade.price_precision, self.precision_mode)
         order = Order(
             id=self.order_id_counter,
             ft_trade_id=trade.id,
@@ -674,12 +682,12 @@ class Backtesting:
             side=trade.exit_side,
             order_type=order_type,
             status="open",
-            price=rate,
-            average=rate,
+            price=close_rate,
+            average=close_rate,
             amount=amount,
             filled=0,
             remaining=amount,
-            cost=amount * rate,
+            cost=amount * close_rate,
         )
         trade.orders.append(order)
         return trade
@@ -726,18 +734,21 @@ class Backtesting:
     def get_valid_price_and_stake(
         self, pair: str, row: Tuple, propose_rate: float, stake_amount: float,
         direction: LongShort, current_time: datetime, entry_tag: Optional[str],
-        trade: Optional[LocalTrade], order_type: str
+        trade: Optional[LocalTrade], order_type: str, price_precision: Optional[float]
     ) -> Tuple[float, float, float, float]:
 
         if order_type == 'limit':
-            propose_rate = strategy_safe_wrapper(self.strategy.custom_entry_price,
-                                                 default_retval=propose_rate)(
+            new_rate = strategy_safe_wrapper(self.strategy.custom_entry_price,
+                                             default_retval=propose_rate)(
                 pair=pair, current_time=current_time,
                 proposed_rate=propose_rate, entry_tag=entry_tag,
                 side=direction,
             )  # default value is the open rate
             # We can't place orders higher than current high (otherwise it'd be a stop limit entry)
             # which freqtrade does not support in live.
+            if new_rate != propose_rate:
+                propose_rate = price_to_precision(new_rate, price_precision,
+                                                  self.precision_mode)
             if direction == "short":
                 propose_rate = max(propose_rate, row[LOW_IDX])
             else:
@@ -799,9 +810,11 @@ class Backtesting:
         pos_adjust = trade is not None and requested_rate is None
 
         stake_amount_ = stake_amount or (trade.stake_amount if trade else 0.0)
+        precision_price = self.exchange.get_precision_price(pair)
+
         propose_rate, stake_amount, leverage, min_stake_amount = self.get_valid_price_and_stake(
             pair, row, row[OPEN_IDX], stake_amount_, direction, current_time, entry_tag, trade,
-            order_type
+            order_type, precision_price,
         )
 
         # replace proposed rate if another rate was requested
@@ -817,8 +830,6 @@ class Backtesting:
         if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount):
             self.order_id_counter += 1
             base_currency = self.exchange.get_pair_base_currency(pair)
-            precision_price = self.exchange.get_precision_price(pair)
-            propose_rate = price_to_precision(propose_rate, precision_price, self.precision_mode)
             amount_p = (stake_amount / propose_rate) * leverage
 
             contract_size = self.exchange.get_contract_size(pair)
@@ -914,30 +925,23 @@ class Backtesting:
         return trade
 
     def handle_left_open(self, open_trades: Dict[str, List[LocalTrade]],
-                         data: Dict[str, List[Tuple]]) -> List[LocalTrade]:
+                         data: Dict[str, List[Tuple]]) -> None:
         """
         Handling of left open trades at the end of backtesting
         """
-        trades = []
         for pair in open_trades.keys():
-            if len(open_trades[pair]) > 0:
-                for trade in open_trades[pair]:
-                    if trade.open_order_id and trade.nr_of_successful_entries == 0:
-                        # Ignore trade if entry-order did not fill yet
-                        continue
-                    exit_row = data[pair][-1]
-                    self._exit_trade(trade, exit_row, exit_row[OPEN_IDX], trade.amount)
-                    trade.orders[-1].close_bt_order(exit_row[DATE_IDX].to_pydatetime(), trade)
+            for trade in list(open_trades[pair]):
+                if trade.open_order_id and trade.nr_of_successful_entries == 0:
+                    # Ignore trade if entry-order did not fill yet
+                    continue
+                exit_row = data[pair][-1]
+                self._exit_trade(trade, exit_row, exit_row[OPEN_IDX], trade.amount)
+                trade.orders[-1].close_bt_order(exit_row[DATE_IDX].to_pydatetime(), trade)
 
-                    trade.close_date = exit_row[DATE_IDX].to_pydatetime()
-                    trade.exit_reason = ExitType.FORCE_EXIT.value
-                    trade.close(exit_row[OPEN_IDX], show_msg=False)
-                    LocalTrade.close_bt_trade(trade)
-                    # Deepcopy object to have wallets update correctly
-                    trade1 = deepcopy(trade)
-                    trade1.is_open = True
-                    trades.append(trade1)
-        return trades
+                trade.close_date = exit_row[DATE_IDX].to_pydatetime()
+                trade.exit_reason = ExitType.FORCE_EXIT.value
+                trade.close(exit_row[OPEN_IDX], show_msg=False)
+                LocalTrade.close_bt_trade(trade)
 
     def trade_slot_available(self, max_open_trades: int, open_trade_count: int) -> bool:
         # Always allow trades when max_open_trades is enabled.
@@ -961,9 +965,8 @@ class Backtesting:
             return 'short'
         return None
 
-    def run_protections(
-            self, enable_protections, pair: str, current_time: datetime, side: LongShort):
-        if enable_protections:
+    def run_protections(self, pair: str, current_time: datetime, side: LongShort):
+        if self.enable_protections:
             self.protections.stop_per_pair(pair, current_time, side)
             self.protections.global_stop(current_time, side)
 
@@ -1045,7 +1048,7 @@ class Backtesting:
             if requested_rate:
                 self._enter_trade(pair=trade.pair, row=row, trade=trade,
                                   requested_rate=requested_rate,
-                                  requested_stake=(order.remaining * order.price),
+                                  requested_stake=(order.remaining * order.price / trade.leverage),
                                   direction='short' if trade.is_short else 'long')
                 self.replaced_entry_orders += 1
             else:
@@ -1069,10 +1072,78 @@ class Backtesting:
             return None
         return row
 
-    def backtest(self, processed: Dict,  # noqa: max-complexity: 13
+    def backtest_loop(
+            self, row: Tuple, pair: str, current_time: datetime, end_date: datetime,
+            max_open_trades: int, open_trade_count_start: int) -> int:
+        """
+        NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized.
+
+        Backtesting processing for one candle/pair.
+        """
+        for t in list(LocalTrade.bt_trades_open_pp[pair]):
+            # 1. Manage currently open orders of active trades
+            if self.manage_open_orders(t, current_time, row):
+                # Close trade
+                open_trade_count_start -= 1
+                LocalTrade.remove_bt_trade(t)
+                self.wallets.update()
+
+        # 2. Process entries.
+        # without positionstacking, we can only have one open trade per pair.
+        # max_open_trades must be respected
+        # don't open on the last row
+        trade_dir = self.check_for_trade_entry(row)
+        if (
+            (self._position_stacking or len(LocalTrade.bt_trades_open_pp[pair]) == 0)
+            and self.trade_slot_available(max_open_trades, open_trade_count_start)
+            and current_time != end_date
+            and trade_dir is not None
+            and not PairLocks.is_pair_locked(pair, row[DATE_IDX], trade_dir)
+        ):
+            trade = self._enter_trade(pair, row, trade_dir)
+            if trade:
+                # TODO: hacky workaround to avoid opening > max_open_trades
+                # This emulates previous behavior - not sure if this is correct
+                # Prevents entering if the trade-slot was freed in this candle
+                open_trade_count_start += 1
+                # logger.debug(f"{pair} - Emulate creation of new trade: {trade}.")
+                LocalTrade.add_bt_trade(trade)
+                self.wallets.update()
+
+        for trade in list(LocalTrade.bt_trades_open_pp[pair]):
+            # 3. Process entry orders.
+            order = trade.select_order(trade.entry_side, is_open=True)
+            if order and self._get_order_filled(order.price, row):
+                order.close_bt_order(current_time, trade)
+                trade.open_order_id = None
+                self.wallets.update()
+
+                # 4. Create exit orders (if any)
+            if not trade.open_order_id:
+                self._get_exit_trade_entry(trade, row)  # Place exit order if necessary
+
+                # 5. Process exit orders.
+            order = trade.select_order(trade.exit_side, is_open=True)
+            if order and self._get_order_filled(order.price, row):
+                order.close_bt_order(current_time, trade)
+                trade.open_order_id = None
+                sub_trade = order.safe_amount_after_fee != trade.amount
+                if sub_trade:
+                    order.close_bt_order(current_time, trade)
+                    trade.recalc_trade_from_orders()
+                else:
+                    trade.close_date = current_time
+                    trade.close(order.price, show_msg=False)
+
+                    # logger.debug(f"{pair} - Backtesting exit {trade}")
+                    LocalTrade.close_bt_trade(trade)
+                self.wallets.update()
+                self.run_protections(pair, current_time, trade.trade_direction)
+        return open_trade_count_start
+
+    def backtest(self, processed: Dict,
                  start_date: datetime, end_date: datetime,
-                 max_open_trades: int = 0, position_stacking: bool = False,
-                 enable_protections: bool = False) -> Dict[str, Any]:
+                 max_open_trades: int = 0) -> Dict[str, Any]:
         """
         Implement backtesting functionality
 
@@ -1085,12 +1156,9 @@ class Backtesting:
         :param start_date: backtesting timerange start datetime
         :param end_date: backtesting timerange end datetime
         :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited
-        :param position_stacking: do we allow position stacking?
-        :param enable_protections: Should protections be enabled?
         :return: DataFrame with trades (results of backtesting)
         """
-        trades: List[LocalTrade] = []
-        self.prepare_backtest(enable_protections)
+        self.prepare_backtest(self.enable_protections)
         # Ensure wallets are uptodate (important for --strategy-list)
         self.wallets.update()
         # Use dict of lists with data for performance
@@ -1101,15 +1169,12 @@ class Backtesting:
         indexes: Dict = defaultdict(int)
         current_time = start_date + timedelta(minutes=self.timeframe_min)
 
-        open_trades: Dict[str, List[LocalTrade]] = defaultdict(list)
-        open_trade_count = 0
-
         self.progress.init_step(BacktestState.BACKTEST, int(
             (end_date - start_date) / timedelta(minutes=self.timeframe_min)))
 
         # Loop timerange and get candle for each pair at that point in time
         while current_time <= end_date:
-            open_trade_count_start = open_trade_count
+            open_trade_count_start = LocalTrade.bt_open_open_trade_count
             self.check_abort()
             for i, pair in enumerate(data):
                 row_index = indexes[pair]
@@ -1121,81 +1186,17 @@ class Backtesting:
                 indexes[pair] = row_index
                 self.dataprovider._set_dataframe_max_index(row_index)
 
-                for t in list(open_trades[pair]):
-                    # 1. Manage currently open orders of active trades
-                    if self.manage_open_orders(t, current_time, row):
-                        # Close trade
-                        open_trade_count -= 1
-                        open_trades[pair].remove(t)
-                        LocalTrade.trades_open.remove(t)
-                        self.wallets.update()
-
-                # 2. Process entries.
-                # without positionstacking, we can only have one open trade per pair.
-                # max_open_trades must be respected
-                # don't open on the last row
-                trade_dir = self.check_for_trade_entry(row)
-                if (
-                    (position_stacking or len(open_trades[pair]) == 0)
-                    and self.trade_slot_available(max_open_trades, open_trade_count_start)
-                    and current_time != end_date
-                    and trade_dir is not None
-                    and not PairLocks.is_pair_locked(pair, row[DATE_IDX], trade_dir)
-                ):
-                    trade = self._enter_trade(pair, row, trade_dir)
-                    if trade:
-                        # TODO: hacky workaround to avoid opening > max_open_trades
-                        # This emulates previous behavior - not sure if this is correct
-                        # Prevents entering if the trade-slot was freed in this candle
-                        open_trade_count_start += 1
-                        open_trade_count += 1
-                        # logger.debug(f"{pair} - Emulate creation of new trade: {trade}.")
-                        open_trades[pair].append(trade)
-                        LocalTrade.add_bt_trade(trade)
-                        self.wallets.update()
-
-                for trade in list(open_trades[pair]):
-                    # 3. Process entry orders.
-                    order = trade.select_order(trade.entry_side, is_open=True)
-                    if order and self._get_order_filled(order.price, row):
-                        order.close_bt_order(current_time, trade)
-                        trade.open_order_id = None
-                        self.wallets.update()
-
-                    # 4. Create exit orders (if any)
-                    if not trade.open_order_id:
-                        self._get_exit_trade_entry(trade, row)  # Place exit order if necessary
-
-                    # 5. Process exit orders.
-                    order = trade.select_order(trade.exit_side, is_open=True)
-                    if order and self._get_order_filled(order.price, row):
-                        order.close_bt_order(current_time, trade)
-                        trade.open_order_id = None
-                        sub_trade = order.safe_amount_after_fee != trade.amount
-                        if sub_trade:
-                            order.close_bt_order(current_time, trade)
-                            trade.recalc_trade_from_orders()
-                        else:
-                            trade.close_date = current_time
-                            trade.close(order.price, show_msg=False)
-
-                            # logger.debug(f"{pair} - Backtesting exit {trade}")
-                            open_trade_count -= 1
-                            open_trades[pair].remove(trade)
-                            LocalTrade.close_bt_trade(trade)
-                            trades.append(trade)
-                        self.wallets.update()
-                        self.run_protections(
-                            enable_protections, pair, current_time, trade.trade_direction)
+                open_trade_count_start = self.backtest_loop(
+                    row, pair, current_time, end_date, max_open_trades, open_trade_count_start)
 
             # Move time one configured time_interval ahead.
             self.progress.increment()
             current_time += timedelta(minutes=self.timeframe_min)
 
-        trades += self.handle_left_open(open_trades, data=data)
+        self.handle_left_open(LocalTrade.bt_trades_open_pp, data=data)
         self.wallets.update()
 
-        results = trade_list_to_dataframe(trades)
+        results = trade_list_to_dataframe(LocalTrade.trades)
         return {
             'results': results,
             'config': self.strategy.config,
@@ -1248,8 +1249,6 @@ class Backtesting:
             start_date=min_date,
             end_date=max_date,
             max_open_trades=max_open_trades,
-            position_stacking=self.config.get('position_stacking', False),
-            enable_protections=self.config.get('enable_protections', False),
         )
         backtest_end_time = datetime.now(timezone.utc)
         results.update({
@@ -1287,8 +1286,7 @@ class Backtesting:
     def _get_min_cached_backtest_date(self):
         min_backtest_date = None
         backtest_cache_age = self.config.get('backtest_cache', constants.BACKTEST_CACHE_DEFAULT)
-        if self.timerange.stopts == 0 or datetime.fromtimestamp(
-           self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc):
+        if self.timerange.stopts == 0 or self.timerange.stopdt > datetime.now(tz=timezone.utc):
             logger.warning('Backtest result caching disabled due to use of open-ended timerange.')
         elif backtest_cache_age == 'day':
             min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(days=1)

freqtrade/optimize/hyperopt.py

@@ -24,6 +24,7 @@ from pandas import DataFrame
 from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config
 from freqtrade.data.converter import trim_dataframes
 from freqtrade.data.history import get_timerange
+from freqtrade.data.metrics import calculate_market_change
 from freqtrade.enums import HyperoptState
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import deep_merge_dicts, file_dump_json, plural
@@ -111,6 +112,7 @@ class Hyperopt:
 
         self.clean_hyperopt()
 
+        self.market_change = 0.0
         self.num_epochs_saved = 0
         self.current_best_epoch: Optional[Dict[str, Any]] = None
 
@@ -120,7 +122,6 @@ class Hyperopt:
         else:
             logger.debug('Ignoring max_open_trades (--disable-max-market-positions was used) ...')
             self.max_open_trades = 0
-        self.position_stacking = self.config.get('position_stacking', False)
 
         if HyperoptTools.has_space(self.config, 'sell'):
             # Make sure use_exit_signal is enabled
@@ -256,6 +257,7 @@ class Hyperopt:
             logger.debug("Hyperopt has 'protection' space")
             # Enable Protections if protection space is selected.
             self.config['enable_protections'] = True
+            self.backtesting.enable_protections = True
             self.protection_space = self.custom_hyperopt.protection_space()
 
         if HyperoptTools.has_space(self.config, 'buy'):
@@ -337,8 +339,6 @@ class Hyperopt:
             start_date=self.min_date,
             end_date=self.max_date,
             max_open_trades=self.max_open_trades,
-            position_stacking=self.position_stacking,
-            enable_protections=self.config.get('enable_protections', False),
         )
         backtest_end_time = datetime.now(timezone.utc)
         bt_results.update({
@@ -357,7 +357,7 @@ class Hyperopt:
 
         strat_stats = generate_strategy_stats(
             self.pairlist, self.backtesting.strategy.get_strategy_name(),
-            backtesting_results, min_date, max_date, market_change=0
+            backtesting_results, min_date, max_date, market_change=self.market_change
         )
         results_explanation = HyperoptTools.format_results_explanation_string(
             strat_stats, self.config['stake_currency'])
@@ -425,6 +425,9 @@ class Hyperopt:
         # Trim startup period from analyzed dataframe to get correct dates for output.
         trimmed = trim_dataframes(preprocessed, self.timerange, self.backtesting.required_startup)
         self.min_date, self.max_date = get_timerange(trimmed)
+        if not self.market_change:
+            self.market_change = calculate_market_change(trimmed, 'close')
+
         # Real trimming will happen as part of backtesting.
         return preprocessed
 

freqtrade/optimize/hyperopt_tools.py

@@ -12,11 +12,12 @@ import tabulate
 from colorama import Fore, Style
 from pandas import isna, json_normalize
 
-from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES, Config
+from freqtrade.constants import FTHYPT_FILEVERSION, Config
 from freqtrade.enums import HyperoptState
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2
 from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs
+from freqtrade.optimize.optimize_reports import generate_wins_draws_losses
 
 
 logger = logging.getLogger(__name__)
@@ -50,9 +51,8 @@ class HyperoptTools():
         Get Strategy-location (filename) from strategy_name
         """
         from freqtrade.resolvers.strategy_resolver import StrategyResolver
-        directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES))
         strategy_objs = StrategyResolver.search_all_objects(
-            directory, False, config.get('recursive_strategy_search', False))
+            config, False, config.get('recursive_strategy_search', False))
         strategies = [s for s in strategy_objs if s['name'] == strategy_name]
         if strategies:
             strategy = strategies[0]
@@ -326,8 +326,10 @@ class HyperoptTools():
 
         # New mode, using backtest result for metrics
         trials['results_metrics.winsdrawslosses'] = trials.apply(
-            lambda x: f"{x['results_metrics.wins']} {x['results_metrics.draws']:>4} "
-                      f"{x['results_metrics.losses']:>4}", axis=1)
+            lambda x: generate_wins_draws_losses(
+                            x['results_metrics.wins'], x['results_metrics.draws'],
+                            x['results_metrics.losses']
+                      ), axis=1)
 
         trials = trials[['Best', 'current_epoch', 'results_metrics.total_trades',
                          'results_metrics.winsdrawslosses',
@@ -338,7 +340,7 @@ class HyperoptTools():
                          'loss', 'is_initial_point', 'is_random', 'is_best']]
 
         trials.columns = [
-            'Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
+            'Best', 'Epoch', 'Trades', ' Win  Draw  Loss  Win%', 'Avg profit',
             'Total profit', 'Profit', 'Avg duration', 'max_drawdown', 'max_drawdown_account',
             'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_random', 'is_best'
             ]
@@ -468,9 +470,9 @@ class HyperoptTools():
 
         base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades',
                         'results_metrics.profit_mean', 'results_metrics.profit_median',
-                        'results_metrics.profit_total',
-                        'Stake currency',
+                        'results_metrics.profit_total', 'Stake currency',
                         'results_metrics.profit_total_abs', 'results_metrics.holding_avg',
+                        'results_metrics.trade_count_long', 'results_metrics.trade_count_short',
                         'loss', 'is_initial_point', 'is_best']
         perc_multi = 100
 
@@ -478,7 +480,9 @@ class HyperoptTools():
         trials = trials[base_metrics + param_metrics]
 
         base_columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Median profit', 'Total profit',
-                        'Stake currency', 'Profit', 'Avg duration', 'Objective',
+                        'Stake currency', 'Profit', 'Avg duration',
+                        'Trade count long', 'Trade count short',
+                        'Objective',
                         'is_initial_point', 'is_best']
         param_columns = list(results[0]['params_dict'].keys())
         trials.columns = base_columns + param_columns

freqtrade/optimize/optimize_reports.py

@@ -86,7 +86,7 @@ def _get_line_header(first_column: str, stake_currency: str,
             'Win  Draw  Loss  Win%']
 
 
-def _generate_wins_draws_losses(wins, draws, losses):
+def generate_wins_draws_losses(wins, draws, losses):
     if wins > 0 and losses == 0:
         wl_ratio = '100'
     elif wins == 0:
@@ -408,10 +408,10 @@ def generate_strategy_stats(pairlist: List[str],
 
     exit_reason_stats = generate_exit_reason_stats(max_open_trades=max_open_trades,
                                                    results=results)
-    left_open_results = generate_pair_metrics(pairlist, stake_currency=stake_currency,
-                                              starting_balance=start_balance,
-                                              results=results.loc[results['is_open']],
-                                              skip_nan=True)
+    left_open_results = generate_pair_metrics(
+        pairlist, stake_currency=stake_currency, starting_balance=start_balance,
+        results=results.loc[results['exit_reason'] == 'force_exit'], skip_nan=True)
+
     daily_stats = generate_daily_stats(results)
     trade_stats = generate_trading_stats(results)
     best_pair = max([pair for pair in pair_results if pair['key'] != 'TOTAL'],
@@ -600,7 +600,7 @@ def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: st
     output = [[
         t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'],
         t['profit_total_pct'], t['duration_avg'],
-        _generate_wins_draws_losses(t['wins'], t['draws'], t['losses'])
+        generate_wins_draws_losses(t['wins'], t['draws'], t['losses'])
     ] for t in pair_results]
     # Ignore type as floatfmt does allow tuples but mypy does not know that
     return tabulate(output, headers=headers,
@@ -626,7 +626,7 @@ def text_table_exit_reason(exit_reason_stats: List[Dict[str, Any]], stake_curren
 
     output = [[
         t.get('exit_reason', t.get('sell_reason')), t['trades'],
-        _generate_wins_draws_losses(t['wins'], t['draws'], t['losses']),
+        generate_wins_draws_losses(t['wins'], t['draws'], t['losses']),
         t['profit_mean_pct'], t['profit_sum_pct'],
         round_coin_value(t['profit_total_abs'], stake_currency, False),
         t['profit_total_pct'],
@@ -656,7 +656,7 @@ def text_table_tags(tag_type: str, tag_results: List[Dict[str, Any]], stake_curr
             t['profit_total_abs'],
             t['profit_total_pct'],
             t['duration_avg'],
-            _generate_wins_draws_losses(
+            generate_wins_draws_losses(
                 t['wins'],
                 t['draws'],
                 t['losses'])] for t in tag_results]
@@ -715,7 +715,7 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str:
     output = [[
         t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'],
         t['profit_total_pct'], t['duration_avg'],
-        _generate_wins_draws_losses(t['wins'], t['draws'], t['losses']), drawdown]
+        generate_wins_draws_losses(t['wins'], t['draws'], t['losses']), drawdown]
         for t, drawdown in zip(strategy_results, drawdown)]
     # Ignore type as floatfmt does allow tuples but mypy does not know that
     return tabulate(output, headers=headers,

freqtrade/persistence/migrations.py

@@ -1,5 +1,5 @@
 import logging
-from typing import List
+from typing import List, Optional
 
 from sqlalchemy import inspect, select, text, tuple_, update
 
@@ -31,9 +31,9 @@ def get_backup_name(tabs: List[str], backup_prefix: str):
     return table_back_name
 
 
-def get_last_sequence_ids(engine, trade_back_name, order_back_name):
-    order_id: int = None
-    trade_id: int = None
+def get_last_sequence_ids(engine, trade_back_name: str, order_back_name: str):
+    order_id: Optional[int] = None
+    trade_id: Optional[int] = None
 
     if engine.name == 'postgresql':
         with engine.begin() as connection:

freqtrade/persistence/trade_model.py

@@ -2,6 +2,7 @@
 This module contains the class to persist trades into SQLite
 """
 import logging
+from collections import defaultdict
 from datetime import datetime, timedelta, timezone
 from math import isclose
 from typing import Any, Dict, List, Optional
@@ -89,6 +90,13 @@ class Order(_DECL_BASE):
     def safe_filled(self) -> float:
         return self.filled if self.filled is not None else self.amount or 0.0
 
+    @property
+    def safe_remaining(self) -> float:
+        return (
+            self.remaining if self.remaining is not None else
+            self.amount - (self.filled or 0.0)
+        )
+
     @property
     def safe_fee_base(self) -> float:
         return self.ft_fee_base or 0.0
@@ -255,6 +263,9 @@ class LocalTrade():
     # Trades container for backtesting
     trades: List['LocalTrade'] = []
     trades_open: List['LocalTrade'] = []
+    # Copy of trades_open - but indexed by pair
+    bt_trades_open_pp: Dict[str, List['LocalTrade']] = defaultdict(list)
+    bt_open_open_trade_count: int = 0
     total_profit: float = 0
     realized_profit: float = 0
 
@@ -538,6 +549,8 @@ class LocalTrade():
         """
         LocalTrade.trades = []
         LocalTrade.trades_open = []
+        LocalTrade.bt_trades_open_pp = defaultdict(list)
+        LocalTrade.bt_open_open_trade_count = 0
         LocalTrade.total_profit = 0
 
     def adjust_min_max_rates(self, current_price: float, current_price_low: float) -> None:
@@ -661,7 +674,7 @@ class LocalTrade():
                 self.close(order.safe_price)
             else:
                 self.recalc_trade_from_orders()
-        elif order.ft_order_side == 'stoploss':
+        elif order.ft_order_side == 'stoploss' and order.status not in ('canceled', 'open'):
             self.stoploss_order_id = None
             self.close_rate_requested = self.stop_loss
             self.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
@@ -1067,6 +1080,8 @@ class LocalTrade():
     @staticmethod
     def close_bt_trade(trade):
         LocalTrade.trades_open.remove(trade)
+        LocalTrade.bt_trades_open_pp[trade.pair].remove(trade)
+        LocalTrade.bt_open_open_trade_count -= 1
         LocalTrade.trades.append(trade)
         LocalTrade.total_profit += trade.close_profit_abs
 
@@ -1074,9 +1089,17 @@ class LocalTrade():
     def add_bt_trade(trade):
         if trade.is_open:
             LocalTrade.trades_open.append(trade)
+            LocalTrade.bt_trades_open_pp[trade.pair].append(trade)
+            LocalTrade.bt_open_open_trade_count += 1
         else:
             LocalTrade.trades.append(trade)
 
+    @staticmethod
+    def remove_bt_trade(trade):
+        LocalTrade.trades_open.remove(trade)
+        LocalTrade.bt_trades_open_pp[trade.pair].remove(trade)
+        LocalTrade.bt_open_open_trade_count -= 1
+
     @staticmethod
     def get_open_trades() -> List[Any]:
         """
@@ -1092,7 +1115,7 @@ class LocalTrade():
         if Trade.use_db:
             return Trade.query.filter(Trade.is_open.is_(True)).count()
         else:
-            return len(LocalTrade.trades_open)
+            return LocalTrade.bt_open_open_trade_count
 
     @staticmethod
     def stoploss_reinitialization(desired_stoploss):
@@ -1128,7 +1151,8 @@ class Trade(_DECL_BASE, LocalTrade):
 
     id = Column(Integer, primary_key=True)
 
-    orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan", lazy="joined")
+    orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan",
+                          lazy="selectin", innerjoin=True)
 
     exchange = Column(String(25), nullable=False)
     pair = Column(String(25), nullable=False, index=True)
@@ -1504,3 +1528,87 @@ class Trade(_DECL_BASE, LocalTrade):
             Order.status == 'closed'
         ).scalar()
         return trading_volume
+
+    @staticmethod
+    def from_json(json_str: str) -> 'Trade':
+        """
+        Create a Trade instance from a json string.
+
+        Used for debugging purposes - please keep.
+        :param json_str: json string to parse
+        :return: Trade instance
+        """
+        import rapidjson
+        data = rapidjson.loads(json_str)
+        trade = Trade(
+            id=data["trade_id"],
+            pair=data["pair"],
+            base_currency=data["base_currency"],
+            stake_currency=data["quote_currency"],
+            is_open=data["is_open"],
+            exchange=data["exchange"],
+            amount=data["amount"],
+            amount_requested=data["amount_requested"],
+            stake_amount=data["stake_amount"],
+            strategy=data["strategy"],
+            enter_tag=data["enter_tag"],
+            timeframe=data["timeframe"],
+            fee_open=data["fee_open"],
+            fee_open_cost=data["fee_open_cost"],
+            fee_open_currency=data["fee_open_currency"],
+            fee_close=data["fee_close"],
+            fee_close_cost=data["fee_close_cost"],
+            fee_close_currency=data["fee_close_currency"],
+            open_date=datetime.fromtimestamp(data["open_timestamp"] // 1000, tz=timezone.utc),
+            open_rate=data["open_rate"],
+            open_rate_requested=data["open_rate_requested"],
+            open_trade_value=data["open_trade_value"],
+            close_date=(datetime.fromtimestamp(data["close_timestamp"] // 1000, tz=timezone.utc)
+                        if data["close_timestamp"] else None),
+            realized_profit=data["realized_profit"],
+            close_rate=data["close_rate"],
+            close_rate_requested=data["close_rate_requested"],
+            close_profit=data["close_profit"],
+            close_profit_abs=data["close_profit_abs"],
+            exit_reason=data["exit_reason"],
+            exit_order_status=data["exit_order_status"],
+            stop_loss=data["stop_loss_abs"],
+            stop_loss_pct=data["stop_loss_ratio"],
+            stoploss_order_id=data["stoploss_order_id"],
+            stoploss_last_update=(datetime.fromtimestamp(data["stoploss_last_update"] // 1000,
+                                  tz=timezone.utc) if data["stoploss_last_update"] else None),
+            initial_stop_loss=data["initial_stop_loss_abs"],
+            initial_stop_loss_pct=data["initial_stop_loss_ratio"],
+            min_rate=data["min_rate"],
+            max_rate=data["max_rate"],
+            leverage=data["leverage"],
+            interest_rate=data["interest_rate"],
+            liquidation_price=data["liquidation_price"],
+            is_short=data["is_short"],
+            trading_mode=data["trading_mode"],
+            funding_fees=data["funding_fees"],
+            open_order_id=data["open_order_id"],
+        )
+        for order in data["orders"]:
+
+            order_obj = Order(
+                amount=order["amount"],
+                ft_order_side=order["ft_order_side"],
+                ft_pair=order["pair"],
+                ft_is_open=order["is_open"],
+                order_id=order["order_id"],
+                status=order["status"],
+                average=order["average"],
+                cost=order["cost"],
+                filled=order["filled"],
+                order_date=datetime.strptime(order["order_date"], DATETIME_PRINT_FORMAT),
+                order_filled_date=(datetime.fromtimestamp(
+                    order["order_filled_timestamp"] // 1000, tz=timezone.utc)
+                    if order["order_filled_timestamp"] else None),
+                order_type=order["order_type"],
+                price=order["price"],
+                remaining=order["remaining"],
+            )
+            trade.orders.append(order_obj)
+
+        return trade