Merge branch 'freqtrade:develop' into strategy_utils

.github/workflows/ci.yml

@@ -23,8 +23,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-18.04, ubuntu-20.04, ubuntu-22.04 ]
+        os: [ ubuntu-20.04, ubuntu-22.04 ]
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v3
@@ -90,14 +90,14 @@ jobs:
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 6 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
-    - name: Flake8
-      run: |
-        flake8
-
     - name: Sort imports (isort)
       run: |
         isort --check .
 
+    - name: Run Ruff
+      run: |
+        ruff check --format=github .
+
     - name: Mypy
       run: |
         mypy freqtrade scripts tests
@@ -115,7 +115,7 @@ jobs:
     strategy:
       matrix:
         os: [ macos-latest ]
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v3
@@ -186,14 +186,14 @@ jobs:
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
-    - name: Flake8
-      run: |
-        flake8
-
     - name: Sort imports (isort)
       run: |
         isort --check .
 
+    - name: Run Ruff
+      run: |
+        ruff check --format=github .
+
     - name: Mypy
       run: |
         mypy freqtrade scripts
@@ -212,7 +212,7 @@ jobs:
     strategy:
       matrix:
         os: [ windows-latest ]
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v3
@@ -248,9 +248,9 @@ jobs:
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
-    - name: Flake8
+    - name: Run Ruff
       run: |
-        flake8
+        ruff check --format=github .
 
     - name: Mypy
       run: |

.pre-commit-config.yaml

@@ -8,16 +8,17 @@ repos:
         # stages: [push]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v0.991"
+    rev: "v1.0.1"
     hooks:
       - id: mypy
         exclude: build_helpers
         additional_dependencies:
-          - types-cachetools==5.3.0.0
+          - types-cachetools==5.3.0.4
           - types-filelock==3.2.7
-          - types-requests==2.28.11.12
+          - types-requests==2.28.11.15
-          - types-tabulate==0.9.0.0
+          - types-tabulate==0.9.0.1
-          - types-python-dateutil==2.8.19.6
+          - types-python-dateutil==2.8.19.9
+          - SQLAlchemy==2.0.4
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort
@@ -27,6 +28,12 @@ repos:
         name: isort (python)
         # stages: [push]
 
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.251'
+    hooks:
+      - id: ruff
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:

CONTRIBUTING.md

@@ -45,16 +45,17 @@ pytest tests/test_<file_name>.py::test_<method_name>
 
 ### 2. Test if your code is PEP8 compliant
 
-#### Run Flake8
+#### Run Ruff
 
 ```bash
-flake8 freqtrade tests scripts
+ruff .
 ```
 
-We receive a lot of code that fails the `flake8` checks.
+We receive a lot of code that fails the `ruff` checks.
 To help with that, we encourage you to install the git pre-commit 
 hook that will warn you when you try to commit code that fails these checks.
-Guide for installing them is [here](http://flake8.pycqa.org/en/latest/user/using-hooks.html).
+
+you can manually run pre-commit with `pre-commit run -a`.
 
 ##### Additional styles applied
 

build_helpers/pre_commit_update.py

@@ -8,12 +8,17 @@ import yaml
 
 pre_commit_file = Path('.pre-commit-config.yaml')
 require_dev = Path('requirements-dev.txt')
+require = Path('requirements.txt')
 
 with require_dev.open('r') as rfile:
     requirements = rfile.readlines()
 
+with require.open('r') as rfile:
+    requirements.extend(rfile.readlines())
+
 # Extract types only
-type_reqs = [r.strip('\n') for r in requirements if r.startswith('types-')]
+type_reqs = [r.strip('\n') for r in requirements if r.startswith(
+    'types-') or r.startswith('SQLAlchemy')]
 
 with pre_commit_file.open('r') as file:
     f = yaml.load(file, Loader=yaml.FullLoader)

config_examples/config_freqai.example.json

@@ -48,7 +48,7 @@
     ],
     "freqai": {
         "enabled": true,
-        "purge_old_models": true,
+        "purge_old_models": 2,
         "train_period_days": 15,
         "backtest_period_days": 7,
         "live_retrain_hours": 0,

docs/advanced-setup.md

@@ -192,7 +192,7 @@ $RepeatedMsgReduction on
 
 ### Logging to journald
 
-This needs the `systemd` python package installed as the dependency, which is not available on Windows. Hence, the whole journald logging functionality is not available for a bot running on Windows.
+This needs the `cysystemd` python package installed as dependency (`pip install cysystemd`), which is not available on Windows. Hence, the whole journald logging functionality is not available for a bot running on Windows.
 
 To send Freqtrade log messages to `journald` system service use the `--logfile` command line option with the value in the following format:
 

docs/deprecated.md

@@ -74,3 +74,8 @@ Webhook terminology changed from "sell" to "exit", and from "buy" to "entry", re
 * `webhooksell`, `webhookexit` -> `exit`
 * `webhooksellfill`, `webhookexitfill` -> `exit_fill`
 * `webhooksellcancel`, `webhookexitcancel` -> `exit_cancel`
+
+
+## Removal of `populate_any_indicators`
+
+version 2023.3 saw the removal of `populate_any_indicators` in favor of split methods for feature engineering and targets. Please read the [migration document](strategy_migration.md#freqai-strategy) for full details.

docs/developer.md

@@ -24,7 +24,7 @@ This will spin up a local server (usually on port 8000) so you can see if everyt
 To configure a development environment, you can either use the provided [DevContainer](#devcontainer-setup), or use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
 Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`.
 
-This will install all required tools for development, including `pytest`, `flake8`, `mypy`, and `coveralls`.
+This will install all required tools for development, including `pytest`, `ruff`, `mypy`, and `coveralls`.
 
 Then install the git hook scripts by running `pre-commit install`, so your changes will be verified locally before committing.
 This avoids a lot of waiting for CI already, as some basic formatting checks are done locally on your machine.

docs/faq.md

@@ -2,7 +2,7 @@
 
 ## Supported Markets
 
-Freqtrade supports spot trading only.
+Freqtrade supports spot trading, as well as (isolated) futures trading for some selected exchanges. Please refer to the [documentation start page](index.md#supported-futures-exchanges-experimental) for an uptodate list of supported exchanges.
 
 ### Can my bot open short positions?
 
@@ -248,8 +248,26 @@ The Edge module is mostly a result of brainstorming of [@mishaker](https://githu
 You can find further info on expectancy, win rate, risk management and position size in the following sources:
 
 - https://www.tradeciety.com/ultimate-math-guide-for-traders/
-- http://www.vantharp.com/tharp-concepts/expectancy.asp
 - https://samuraitradingacademy.com/trading-expectancy/
 - https://www.learningmarkets.com/determining-expectancy-in-your-trading/
-- http://www.lonestocktrader.com/make-money-trading-positive-expectancy/
+- https://www.lonestocktrader.com/make-money-trading-positive-expectancy/
 - https://www.babypips.com/trading/trade-expectancy-matter
+
+## Official channels
+
+Freqtrade is using exclusively the following official channels:
+
+* [Freqtrade discord server](https://discord.gg/p7nuUNVfP7)
+* [Freqtrade documentation (https://freqtrade.io)](https://freqtrade.io)
+* [Freqtrade github organization](https://github.com/freqtrade)
+
+Nobody affiliated with the freqtrade project will ask you about your exchange keys or anything else exposing your funds to exploitation.
+Should you be asked to expose your exchange keys or send funds to some random wallet, then please don't follow these instructions.
+
+Failing to follow these guidelines will not be responsibility of freqtrade.
+
+## "Freqtrade token"
+
+Freqtrade does not have a Crypto token offering.
+
+Token offerings you find on the internet referring Freqtrade, FreqAI or freqUI must be considered to be a scam, trying to exploit freqtrade's popularity for their own, nefarious gains.

docs/freqai-configuration.md

@@ -9,7 +9,7 @@ FreqAI is configured through the typical [Freqtrade config file](configuration.m
 ```json
     "freqai": {
         "enabled": true,
-        "purge_old_models": true,
+        "purge_old_models": 2,
         "train_period_days": 30,
         "backtest_period_days": 7,
         "identifier" : "unique-id",

docs/freqai-parameter-table.md

@@ -15,7 +15,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
 | `identifier` | **Required.** <br> A unique ID for the current model. If models are saved to disk, the `identifier` allows for reloading specific pre-trained models/data. <br> **Datatype:** String.
 | `live_retrain_hours` | Frequency of retraining during dry/live runs. <br> **Datatype:** Float > 0. <br> Default: `0` (models retrain as often as possible).
 | `expiration_hours` | Avoid making predictions if a model is more than `expiration_hours` old. <br> **Datatype:** Positive integer. <br> Default: `0` (models never expire).
-| `purge_old_models` | Delete all unused models during live runs (not relevant to backtesting). If set to false (not default), dry/live runs will accumulate all unused models to disk. If <br> **Datatype:** Boolean. <br> Default: `True`.
+| `purge_old_models` | Number of models to keep on disk (not relevant to backtesting). Default is 2, which means that dry/live runs will keep the latest 2 models on disk. Setting to 0 keeps all models. This parameter also accepts a boolean to maintain backwards compatibility. <br> **Datatype:** Integer. <br> Default: `2`.
 | `save_backtest_models` | Save models to disk when running backtesting. Backtesting operates most efficiently by saving the prediction data and reusing them directly for subsequent runs (when you wish to tune entry/exit parameters). Saving backtesting models to disk also allows to use the same model files for starting a dry/live instance with the same model `identifier`. <br> **Datatype:** Boolean. <br> Default: `False` (no models are saved).
 | `fit_live_predictions_candles` | Number of historical candles to use for computing target (label) statistics from prediction data, instead of from the training dataset (more information can be found [here](freqai-configuration.md#creating-a-dynamic-target-threshold)). <br> **Datatype:** Positive integer.
 | `continual_learning` | Use the final state of the most recently trained model as starting point for the new model, allowing for incremental learning (more information can be found [here](freqai-running.md#continual-learning)). <br> **Datatype:** Boolean. <br> Default: `False`.
@@ -45,6 +45,8 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
 | `noise_standard_deviation` | If set, FreqAI adds noise to the training features with the aim of preventing overfitting. FreqAI generates random deviates from a gaussian distribution with a standard deviation of `noise_standard_deviation` and adds them to all data points. `noise_standard_deviation` should be kept relative to the normalized space, i.e., between -1 and 1. In other words, since data in FreqAI is always normalized to be between -1 and 1, `noise_standard_deviation: 0.05` would result in 32% of the data being randomly increased/decreased by more than 2.5% (i.e., the percent of data falling within the first standard deviation). <br> **Datatype:** Integer. <br> Default: `0`.
 | `outlier_protection_percentage` | Enable to prevent outlier detection methods from discarding too much data. If more than `outlier_protection_percentage` % of points are detected as outliers by the SVM or DBSCAN, FreqAI will log a warning message and ignore outlier detection, i.e., the original dataset will be kept intact. If the outlier protection is triggered, no predictions will be made based on the training dataset. <br> **Datatype:** Float. <br> Default: `30`.
 | `reverse_train_test_order` | Split the feature dataset (see below) and use the latest data split for training and test on historical split of the data. This allows the model to be trained up to the most recent data point, while avoiding overfitting. However, you should be careful to understand the unorthodox nature of this parameter before employing it. <br> **Datatype:** Boolean. <br> Default: `False` (no reversal).
+| `shuffle_after_split` | Split the data into train and test sets, and then shuffle both sets individually. <br> **Datatype:** Boolean. <br> Default: `False`.
+| `buffer_train_data_candles` | Cut `buffer_train_data_candles` off the beginning and end of the training data *after* the indicators were populated. The main example use is when predicting maxima and minima, the argrelextrema function  cannot know the maxima/minima at the edges of the timerange. To improve model accuracy, it is best to compute argrelextrema on the full timerange and then use this function to cut off the edges (buffer) by the kernel. In another case, if the targets are set to a shifted price movement, this buffer is unnecessary because the shifted candles at the end of the timerange will be NaN and FreqAI will automatically cut those off of the training dataset.<br> **Datatype:** Boolean. <br> Default: `False`.
 
 ### Data split parameters
 

docs/freqai-reinforcement-learning.md

@@ -176,18 +176,19 @@ As you begin to modify the strategy and the prediction model, you will quickly r
 
                 factor = 100
 
-            # you can use feature values from dataframe
+                # you can use feature values from dataframe
-            rsi_now = self.raw_features[f"%-rsi-period-10_shift-1_{self.pair}_"
+                # Assumes the shifted RSI indicator has been generated in the strategy.
-                              f"{self.config['timeframe']}"].iloc[self._current_tick]
+                rsi_now = self.raw_features[f"%-rsi-period-10_shift-1_{self.pair}_"
+                                f"{self.config['timeframe']}"].iloc[self._current_tick]
 
-            # reward agent for entering trades
+                # reward agent for entering trades
-            if (action in (Actions.Long_enter.value, Actions.Short_enter.value)
+                if (action in (Actions.Long_enter.value, Actions.Short_enter.value)
-                    and self._position == Positions.Neutral):
+                        and self._position == Positions.Neutral):
-                if rsi_now < 40:
+                    if rsi_now < 40:
-                    factor = 40 / rsi_now
+                        factor = 40 / rsi_now
-                else:
+                    else:
-                    factor = 1
+                        factor = 1
-                return 25 * factor
+                    return 25 * factor
 
                 # discourage agent from not entering trades
                 if action == Actions.Neutral.value and self._position == Positions.Neutral:

docs/freqai.md

@@ -6,6 +6,9 @@
 
 FreqAI is a software designed to automate a variety of tasks associated with training a predictive machine learning model to generate market forecasts given a set of input signals. In general, FreqAI aims to be a sandbox for easily deploying robust machine learning libraries on real-time data ([details](#freqai-position-in-open-source-machine-learning-landscape)).
 
+!!! Note
+    FreqAI is, and always will be, a not-for-profit, open-source project. FreqAI does *not* have a crypto token, FreqAI does *not* sell signals, and FreqAI does not have a domain besides the present [freqtrade documentation](https://www.freqtrade.io/en/latest/freqai/).
+
 Features include:
 
 * **Self-adaptive retraining** - Retrain models during [live deployments](freqai-running.md#live-deployments) to self-adapt to the market in a supervised manner
@@ -19,7 +22,7 @@ Features include:
 * **Automatic data download** - Compute timeranges for data downloads and update historic data (in live deployments)
 * **Cleaning of incoming data** - Handle NaNs safely before training and model inferencing
 * **Dimensionality reduction** - Reduce the size of the training data via [Principal Component Analysis](freqai-feature-engineering.md#data-dimensionality-reduction-with-principal-component-analysis)
-* **Deploying bot fleets** - Set one bot to train models while a fleet of [follower bots](freqai-running.md#setting-up-a-follower) inference the models and handle trades
+* **Deploying bot fleets** - Set one bot to train models while a fleet of [consumers](producer-consumer.md) use signals.
 
 ## Quick start
 
@@ -68,6 +71,10 @@ pip install -r requirements-freqai.txt
 !!! Note
     Catboost will not be installed on arm devices (raspberry, Mac M1, ARM based VPS, ...), since it does not provide wheels for this platform.
 
+!!! Note "python 3.11"
+    Some dependencies (Catboost, Torch) currently don't support python 3.11. Freqtrade therefore only supports python 3.10 for these models/dependencies.
+    Tests involving these dependencies are skipped on 3.11.
+
 ### Usage with docker
 
 If you are using docker, a dedicated tag with FreqAI dependencies is available as `:freqai`. As such - you can replace the image line in your docker compose file with `image: freqtradeorg/freqtrade:develop_freqai`. This image contains the regular FreqAI dependencies. Similar to native installs, Catboost will not be available on ARM based devices.

docs/installation.md

@@ -290,10 +290,8 @@ cd freqtrade
 
 #### Freqtrade install: Conda Environment
 
-Prepare conda-freqtrade environment, using file `environment.yml`, which exist in main freqtrade directory
-
 ```bash
-conda env create -n freqtrade-conda -f environment.yml
+conda create --name freqtrade python=3.10
 ```
 
 !!! Note "Creating Conda Environment"
@@ -302,12 +300,9 @@ conda env create -n freqtrade-conda -f environment.yml
     ```bash
     # choose your own packages
     conda env create -n [name of the environment] [python version] [packages]
-
-    # point to file with packages
-    conda env create -n [name of the environment] -f [file]
     ```
 
-#### Enter/exit freqtrade-conda environment
+#### Enter/exit freqtrade environment
 
 To check available environments, type
 
@@ -319,7 +314,7 @@ Enter installed environment
 
 ```bash
 # enter conda environment
-conda activate freqtrade-conda
+conda activate freqtrade
 
 # exit conda environment - don't do it now
 conda deactivate
@@ -329,6 +324,7 @@ Install last python dependencies with pip
 
 ```bash
 python3 -m pip install --upgrade pip
+python3 -m pip install -r requirements.txt
 python3 -m pip install -e .
 ```
 
@@ -336,7 +332,7 @@ Patch conda libta-lib (Linux only)
 
 ```bash
 # Ensure that the environment is active!
-conda activate freqtrade-conda
+conda activate freqtrade
 
 cd build_helpers
 bash install_ta-lib.sh ${CONDA_PREFIX} nosudo
@@ -355,8 +351,8 @@ conda env list
 # activate base environment
 conda activate
 
-# activate freqtrade-conda environment
+# activate freqtrade environment
-conda activate freqtrade-conda
+conda activate freqtrade
 
 #deactivate any conda environments
 conda deactivate                              

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.3.7
 mkdocs==1.4.2
-mkdocs-material==9.0.12
+mkdocs-material==9.0.15
 mdx_truly_sane_lists==1.3
 pymdown-extensions==9.9.2
 jinja2==3.1.2

docs/strategy-customization.md

@@ -954,12 +954,14 @@ In some situations it may be confusing to deal with stops relative to current ra
 
 ## Additional data (Wallets)
 
-The strategy provides access to the `Wallets` object. This contains the current balances on the exchange.
+The strategy provides access to the `wallets` object. This contains the current balances on the exchange.
 
-!!! Note
+!!! Note "Backtesting / Hyperopt"
-    Wallets is not available during backtesting / hyperopt.
+    Wallets behaves differently depending on the function it's called.
+    Within `populate_*()` methods, it'll return the full wallet as configured.
+    Within [callbacks](strategy-callbacks.md), you'll get the wallet state corresponding to the actual simulated wallet at that point in the simulation process.
 
-Please always check if `Wallets` is available to avoid failures during backtesting.
+Please always check if `wallets` is available to avoid failures during backtesting.
 
 ``` python
 if self.wallets:

docs/telegram-usage.md

@@ -152,7 +152,7 @@ You can create your own keyboard in `config.json`:
 !!! Note "Supported Commands"
     Only the following commands are allowed. Command arguments are not supported!
 
-    `/start`, `/stop`, `/status`, `/status table`, `/trades`, `/profit`, `/performance`, `/daily`, `/stats`, `/count`, `/locks`, `/balance`, `/stopentry`, `/reload_config`, `/show_config`, `/logs`, `/whitelist`, `/blacklist`, `/edge`, `/help`, `/version`
+    `/start`, `/stop`, `/status`, `/status table`, `/trades`, `/profit`, `/performance`, `/daily`, `/stats`, `/count`, `/locks`, `/balance`, `/stopentry`, `/reload_config`, `/show_config`, `/logs`, `/whitelist`, `/blacklist`, `/edge`, `/help`, `/version`, `/marketdir`
 
 ## Telegram commands
 
@@ -179,6 +179,7 @@ official commands. You can ask at any moment for help with `/help`.
 | `/count` | Displays number of trades used and available
 | `/locks` | Show currently locked pairs.
 | `/unlock <pair or lock_id>` | Remove the lock for this pair (or for this lock id).
+| `/marketdir [long | short | even | none]` | Updates the user managed variable that represents the current market direction. If no direction is provided, the currently set direction will be displayed.
 | **Modify Trade states** |
 | `/forceexit <trade_id> | /fx <tradeid>` | Instantly exits the given trade  (Ignoring `minimum_roi`).
 | `/forceexit all | /fx all` | Instantly exits all open trades (Ignoring `minimum_roi`).
@@ -242,7 +243,7 @@ Enter Tag is configurable via Strategy.
 > **Enter Tag:** Awesome Long Signal
 > **Open Rate:** `0.00007489`
 > **Current Rate:** `0.00007489`
-> **Current Profit:** `12.95%`
+> **Unrealized Profit:** `12.95%`
 > **Stoploss:** `0.00007389 (-0.02%)`
 
 ### /status table
@@ -416,3 +417,27 @@ ARDR/ETH   0.366667      0.143059       -0.01
 ### /version
 
 > **Version:** `0.14.3`
+
+### /marketdir
+
+If a market direction is provided the command updates the user managed variable that represents the current market direction.
+This variable is not set to any valid market direction on bot startup and must be set by the user. The example below is for `/marketdir long`:
+
+```
+Successfully updated marketdirection from none to long.
+```
+
+If no market direction is provided the command outputs the currently set market directions. The example below is for `/marketdir`:
+
+```
+Currently set marketdirection: even
+```
+
+You can use the market direction in your strategy via `self.market_direction`.
+
+!!! Warning "Bot restarts"
+    Please note that the market direction is not persisted, and will be reset after a bot restart/reload.
+
+!!! Danger "Backtesting"
+    As this value/variable is intended to be changed manually in dry/live trading.
+    Strategies using `market_direction` will probably not produce reliable, reproducible results (changes to this variable will not be reflected for backtesting). Use at your own risk.

docs/windows_installation.md

@@ -26,7 +26,7 @@ Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7
 
 As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial pre-compiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which need to be downloaded and installed using `pip install TA_Lib-0.4.25-cp38-cp38-win_amd64.whl` (make sure to use the version matching your python version).
 
-Freqtrade provides these dependencies for the latest 3 Python versions (3.8, 3.9 and 3.10) and for 64bit Windows.
+Freqtrade provides these dependencies for the latest 3 Python versions (3.8, 3.9, 3.10 and 3.11) and for 64bit Windows.
 Other versions must be downloaded from the above link.
 
 ``` powershell

environment.yml

@@ -1,75 +0,0 @@
-name: freqtrade
-channels:
-  - conda-forge
-# - defaults
-dependencies:
-# 1/4 req main
-    - python>=3.8,<=3.10
-    - numpy
-    - pandas
-    - pip
-
-    - py-find-1st
-    - aiohttp
-    - SQLAlchemy
-    - python-telegram-bot<20.0.0
-    - arrow
-    - cachetools
-    - requests
-    - urllib3
-    - jsonschema
-    - TA-Lib
-    - tabulate
-    - jinja2
-    - blosc
-    - sdnotify
-    - fastapi
-    - uvicorn
-    - pyjwt
-    - aiofiles
-    - psutil
-    - colorama
-    - questionary
-    - prompt-toolkit
-    - schedule
-    - python-dateutil
-    - joblib
-    - pyarrow
-
-
-    # ============================
-    # 2/4 req dev
-
-    - coveralls
-    - flake8
-    - mypy
-    - pytest
-    - pytest-asyncio
-    - pytest-cov
-    - pytest-mock
-    - isort
-    - nbconvert
-
-    # ============================
-    # 3/4 req hyperopt
-
-    - scipy
-    - scikit-learn
-    - filelock
-    - scikit-optimize
-    - progressbar2
-    # ============================
-    # 4/4 req plot
-
-    - plotly
-    - jupyter
-
-    - pip:
-        - pycoingecko
-        # - py_find_1st
-        - tables
-        - pytest-random-order
-        - ccxt
-        - flake8-tidy-imports
-        - -e .
-    # - python-rapidjso

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2023.2.dev'
+__version__ = '2023.3.dev'
 
 if 'dev' in __version__:
     from pathlib import Path

freqtrade/commands/data_commands.py

@@ -5,7 +5,7 @@ from datetime import datetime, timedelta
 from typing import Any, Dict, List
 
 from freqtrade.configuration import TimeRange, setup_utils_configuration
-from freqtrade.constants import DATETIME_PRINT_FORMAT
+from freqtrade.constants import DATETIME_PRINT_FORMAT, Config
 from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format
 from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data,
                                     refresh_backtest_trades_data)
@@ -20,15 +20,24 @@ from freqtrade.util.binance_mig import migrate_binance_futures_data
 logger = logging.getLogger(__name__)
 
 
+def _data_download_sanity(config: Config) -> None:
+    if 'days' in config and 'timerange' in config:
+        raise OperationalException("--days and --timerange are mutually exclusive. "
+                                   "You can only specify one or the other.")
+
+    if 'pairs' not in config:
+        raise OperationalException(
+            "Downloading data requires a list of pairs. "
+            "Please check the documentation on how to configure this.")
+
+
 def start_download_data(args: Dict[str, Any]) -> None:
     """
     Download data (former download_backtest_data.py script)
     """
     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
 
-    if 'days' in config and 'timerange' in config:
+    _data_download_sanity(config)
-        raise OperationalException("--days and --timerange are mutually exclusive. "
-                                   "You can only specify one or the other.")
     timerange = TimeRange()
     if 'days' in config:
         time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d")
@@ -40,11 +49,6 @@ def start_download_data(args: Dict[str, Any]) -> None:
     # Remove stake-currency to skip checks which are not relevant for datadownload
     config['stake_currency'] = ''
 
-    if 'pairs' not in config:
-        raise OperationalException(
-            "Downloading data requires a list of pairs. "
-            "Please check the documentation on how to configure this.")
-
     pairs_not_available: List[str] = []
 
     # Init exchange

freqtrade/configuration/load_config.py

@@ -58,7 +58,7 @@ def load_config_file(path: str) -> Dict[str, Any]:
     """
     try:
         # Read config from stdin if requested in the options
-        with open(path) if path != '-' else sys.stdin as file:
+        with Path(path).open() if path != '-' else sys.stdin as file:
             config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE)
     except FileNotFoundError:
         raise OperationalException(

freqtrade/constants.py

@@ -546,7 +546,7 @@ CONF_SCHEMA = {
                 "enabled": {"type": "boolean", "default": False},
                 "keras": {"type": "boolean", "default": False},
                 "write_metrics_to_disk": {"type": "boolean", "default": False},
-                "purge_old_models": {"type": "boolean", "default": True},
+                "purge_old_models": {"type": ["boolean", "number"], "default": 2},
                 "conv_width": {"type": "integer", "default": 1},
                 "train_period_days": {"type": "integer", "default": 0},
                 "backtest_period_days": {"type": "number", "default": 7},
@@ -568,7 +568,9 @@ CONF_SCHEMA = {
                                            "shuffle": {"type": "boolean", "default": False},
                                            "nu": {"type": "number", "default": 0.1}
                                            },
-                                       }
+                                       },
+                        "shuffle_after_split": {"type": "boolean", "default": False},
+                        "buffer_train_data_candles": {"type": "integer", "default": 0}
                     },
                     "required": ["include_timeframes", "include_corr_pairlist", ]
                 },

freqtrade/data/btanalysis.py

@@ -346,7 +346,7 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str,
     return df_final[df_final['open_trades'] > max_open_trades]
 
 
-def trade_list_to_dataframe(trades: List[LocalTrade]) -> pd.DataFrame:
+def trade_list_to_dataframe(trades: Union[List[Trade], List[LocalTrade]]) -> pd.DataFrame:
     """
     Convert list of Trade objects to pandas Dataframe
     :param trades: List of trade objects

freqtrade/data/dataprovider.py

@@ -424,10 +424,8 @@ class DataProvider:
         """
         if self._exchange is None:
             raise OperationalException(NO_EXCHANGE_EXCEPTION)
-        if helping_pairs:
+        final_pairs = (pairlist + helping_pairs) if helping_pairs else pairlist
-            self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs)
+        self._exchange.refresh_latest_ohlcv(final_pairs)
-        else:
-            self._exchange.refresh_latest_ohlcv(pairlist)
 
     @property
     def available_pairs(self) -> ListPairsWithTimeframes:

freqtrade/data/entryexitanalysis.py

@@ -24,9 +24,9 @@ def _load_signal_candles(backtest_dir: Path):
         scpf = Path(backtest_dir.parent / f"{backtest_dir.stem}_signals.pkl")
 
     try:
-        scp = open(scpf, "rb")
+        with scpf.open("rb") as scp:
-        signal_candles = joblib.load(scp)
+            signal_candles = joblib.load(scp)
-        logger.info(f"Loaded signal candles: {str(scpf)}")
+            logger.info(f"Loaded signal candles: {str(scpf)}")
     except Exception as e:
         logger.error("Cannot load signal candles from pickled results: ", e)
 

freqtrade/enums/__init__.py

@@ -5,6 +5,7 @@ from freqtrade.enums.exitchecktuple import ExitCheckTuple
 from freqtrade.enums.exittype import ExitType
 from freqtrade.enums.hyperoptstate import HyperoptState
 from freqtrade.enums.marginmode import MarginMode
+from freqtrade.enums.marketstatetype import MarketDirection
 from freqtrade.enums.ordertypevalue import OrderTypeValues
 from freqtrade.enums.pricetype import PriceType
 from freqtrade.enums.rpcmessagetype import NO_ECHO_MESSAGES, RPCMessageType, RPCRequestType

freqtrade/enums/candletype.py

@@ -13,6 +13,9 @@ class CandleType(str, Enum):
     FUNDING_RATE = "funding_rate"
     # BORROW_RATE = "borrow_rate"  # * unimplemented
 
+    def __str__(self):
+        return f"{self.name.lower()}"
+
     @staticmethod
     def from_string(value: str) -> 'CandleType':
         if not value:

freqtrade/enums/marketstatetype.py

@@ -0,0 +1,15 @@
+from enum import Enum
+
+
+class MarketDirection(Enum):
+    """
+    Enum for various market directions.
+    """
+    LONG = "long"
+    SHORT = "short"
+    EVEN = "even"
+    NONE = "none"
+
+    def __str__(self):
+        # convert to string
+        return self.value

freqtrade/enums/rpcmessagetype.py

@@ -37,5 +37,8 @@ class RPCRequestType(str, Enum):
     WHITELIST = 'whitelist'
     ANALYZED_DF = 'analyzed_df'
 
+    def __str__(self):
+        return self.value
+
 
 NO_ECHO_MESSAGES = (RPCMessageType.ANALYZED_DF, RPCMessageType.WHITELIST, RPCMessageType.NEW_CANDLE)

freqtrade/enums/signaltype.py

@@ -10,6 +10,9 @@ class SignalType(Enum):
     ENTER_SHORT = "enter_short"
     EXIT_SHORT = "exit_short"
 
+    def __str__(self):
+        return f"{self.name.lower()}"
+
 
 class SignalTagType(Enum):
     """
@@ -18,7 +21,13 @@ class SignalTagType(Enum):
     ENTER_TAG = "enter_tag"
     EXIT_TAG = "exit_tag"
 
+    def __str__(self):
+        return f"{self.name.lower()}"
+
 
 class SignalDirection(str, Enum):
     LONG = 'long'
     SHORT = 'short'
+
+    def __str__(self):
+        return f"{self.name.lower()}"

freqtrade/exchange/binance.py

@@ -195,7 +195,7 @@ class Binance(Exchange):
                 leverage_tiers_path = (
                     Path(__file__).parent / 'binance_leverage_tiers.json'
                 )
-                with open(leverage_tiers_path) as json_file:
+                with leverage_tiers_path.open() as json_file:
                     return json_load(json_file)
             else:
                 try:

freqtrade/exchange/exchange.py

@@ -1961,7 +1961,8 @@ class Exchange:
                           cache: bool, drop_incomplete: bool) -> DataFrame:
         # keeping last candle time as last refreshed time of the pair
         if ticks and cache:
-            self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[-1][0] // 1000
+            idx = -2 if drop_incomplete and len(ticks) > 1 else -1
+            self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[idx][0] // 1000
         # keeping parsed dataframe in cache
         ohlcv_df = ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True,
                                       drop_incomplete=drop_incomplete)
@@ -2015,9 +2016,9 @@ class Exchange:
                     continue
                 # Deconstruct tuple (has 5 elements)
                 pair, timeframe, c_type, ticks, drop_hint = res
-                drop_incomplete = drop_hint if drop_incomplete is None else drop_incomplete
+                drop_incomplete_ = drop_hint if drop_incomplete is None else drop_incomplete
                 ohlcv_df = self._process_ohlcv_df(
-                    pair, timeframe, c_type, ticks, cache, drop_incomplete)
+                    pair, timeframe, c_type, ticks, cache, drop_incomplete_)
 
                 results_df[(pair, timeframe, c_type)] = ohlcv_df
 
@@ -2034,7 +2035,9 @@ class Exchange:
         # Timeframe in seconds
         interval_in_sec = timeframe_to_seconds(timeframe)
         plr = self._pairs_last_refresh_time.get((pair, timeframe, candle_type), 0) + interval_in_sec
-        return plr < arrow.utcnow().int_timestamp
+        # current,active candle open date
+        now = int(timeframe_to_prev_date(timeframe).timestamp())
+        return plr < now
 
     @retrier_async
     async def _async_get_candle_history(

freqtrade/exchange/hitbtc.py

@@ -19,5 +19,4 @@ class Hitbtc(Exchange):
 
     _ft_has: Dict = {
         "ohlcv_candle_limit": 1000,
-        "ohlcv_params": {"sort": "DESC"}
     }

freqtrade/exchange/kucoin.py

@@ -64,6 +64,7 @@ class Kucoin(Exchange):
         # ccxt returns status = 'closed' at the moment - which is information ccxt invented.
         # Since we rely on status heavily, we must set it to 'open' here.
         # ref: https://github.com/ccxt/ccxt/pull/16674, (https://github.com/ccxt/ccxt/pull/16553)
-        res['type'] = ordertype
+        if not self._config['dry_run']:
-        res['status'] = 'open'
+            res['type'] = ordertype
+            res['status'] = 'open'
         return res

freqtrade/freqai/data_drawer.py

@@ -72,12 +72,7 @@ class FreqaiDataDrawer:
         self.model_return_values: Dict[str, DataFrame] = {}
         self.historic_data: Dict[str, Dict[str, DataFrame]] = {}
         self.historic_predictions: Dict[str, DataFrame] = {}
-        self.follower_dict: Dict[str, pair_info] = {}
         self.full_path = full_path
-        self.follower_name: str = self.config.get("bot_name", "follower1")
-        self.follower_dict_path = Path(
-            self.full_path / f"follower_dictionary-{self.follower_name}.json"
-        )
         self.historic_predictions_path = Path(self.full_path / "historic_predictions.pkl")
         self.historic_predictions_bkp_path = Path(
             self.full_path / "historic_predictions.backup.pkl")
@@ -131,7 +126,7 @@ class FreqaiDataDrawer:
         """
         exists = self.global_metadata_path.is_file()
         if exists:
-            with open(self.global_metadata_path, "r") as fp:
+            with self.global_metadata_path.open("r") as fp:
                 metatada_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
                 return metatada_dict
         return {}
@@ -144,7 +139,7 @@ class FreqaiDataDrawer:
         """
         exists = self.pair_dictionary_path.is_file()
         if exists:
-            with open(self.pair_dictionary_path, "r") as fp:
+            with self.pair_dictionary_path.open("r") as fp:
                 self.pair_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
         else:
             logger.info("Could not find existing datadrawer, starting from scratch")
@@ -157,7 +152,7 @@ class FreqaiDataDrawer:
         if self.freqai_info.get('write_metrics_to_disk', False):
             exists = self.metric_tracker_path.is_file()
             if exists:
-                with open(self.metric_tracker_path, "r") as fp:
+                with self.metric_tracker_path.open("r") as fp:
                     self.metric_tracker = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
                 logger.info("Loading existing metric tracker from disk.")
             else:
@@ -171,7 +166,7 @@ class FreqaiDataDrawer:
         exists = self.historic_predictions_path.is_file()
         if exists:
             try:
-                with open(self.historic_predictions_path, "rb") as fp:
+                with self.historic_predictions_path.open("rb") as fp:
                     self.historic_predictions = cloudpickle.load(fp)
                 logger.info(
                     f"Found existing historic predictions at {self.full_path}, but beware "
@@ -181,7 +176,7 @@ class FreqaiDataDrawer:
             except EOFError:
                 logger.warning(
                     'Historical prediction file was corrupted. Trying to load backup file.')
-                with open(self.historic_predictions_bkp_path, "rb") as fp:
+                with self.historic_predictions_bkp_path.open("rb") as fp:
                     self.historic_predictions = cloudpickle.load(fp)
                 logger.warning('FreqAI successfully loaded the backup historical predictions file.')
 
@@ -194,7 +189,7 @@ class FreqaiDataDrawer:
         """
         Save historic predictions pickle to disk
         """
-        with open(self.historic_predictions_path, "wb") as fp:
+        with self.historic_predictions_path.open("wb") as fp:
             cloudpickle.dump(self.historic_predictions, fp, protocol=cloudpickle.DEFAULT_PROTOCOL)
 
         # create a backup
@@ -205,33 +200,25 @@ class FreqaiDataDrawer:
         Save metric tracker of all pair metrics collected.
         """
         with self.save_lock:
-            with open(self.metric_tracker_path, 'w') as fp:
+            with self.metric_tracker_path.open('w') as fp:
                 rapidjson.dump(self.metric_tracker, fp, default=self.np_encoder,
                                number_mode=rapidjson.NM_NATIVE)
 
-    def save_drawer_to_disk(self):
+    def save_drawer_to_disk(self) -> None:
         """
         Save data drawer full of all pair model metadata in present model folder.
         """
         with self.save_lock:
-            with open(self.pair_dictionary_path, 'w') as fp:
+            with self.pair_dictionary_path.open('w') as fp:
                 rapidjson.dump(self.pair_dict, fp, default=self.np_encoder,
                                number_mode=rapidjson.NM_NATIVE)
 
-    def save_follower_dict_to_disk(self):
-        """
-        Save follower dictionary to disk (used by strategy for persistent prediction targets)
-        """
-        with open(self.follower_dict_path, "w") as fp:
-            rapidjson.dump(self.follower_dict, fp, default=self.np_encoder,
-                           number_mode=rapidjson.NM_NATIVE)
-
     def save_global_metadata_to_disk(self, metadata: Dict[str, Any]):
         """
         Save global metadata json to disk
         """
         with self.save_lock:
-            with open(self.global_metadata_path, 'w') as fp:
+            with self.global_metadata_path.open('w') as fp:
                 rapidjson.dump(metadata, fp, default=self.np_encoder,
                                number_mode=rapidjson.NM_NATIVE)
 
@@ -239,7 +226,7 @@ class FreqaiDataDrawer:
         if isinstance(object, np.generic):
             return object.item()
 
-    def get_pair_dict_info(self, pair: str) -> Tuple[str, int, bool]:
+    def get_pair_dict_info(self, pair: str) -> Tuple[str, int]:
         """
         Locate and load existing model metadata from persistent storage. If not located,
         create a new one and append the current pair to it and prepare it for its first
@@ -248,12 +235,9 @@ class FreqaiDataDrawer:
         :return:
             model_filename: str = unique filename used for loading persistent objects from disk
             trained_timestamp: int = the last time the coin was trained
-            return_null_array: bool = Follower could not find pair metadata
         """
 
         pair_dict = self.pair_dict.get(pair)
-        # data_path_set = self.pair_dict.get(pair, self.empty_pair_dict).get("data_path", "")
-        return_null_array = False
 
         if pair_dict:
             model_filename = pair_dict["model_filename"]
@@ -263,7 +247,7 @@ class FreqaiDataDrawer:
             model_filename = ""
             trained_timestamp = 0
 
-        return model_filename, trained_timestamp, return_null_array
+        return model_filename, trained_timestamp
 
     def set_pair_dict_info(self, metadata: dict) -> None:
         pair_in_dict = self.pair_dict.get(metadata["pair"])
@@ -382,6 +366,12 @@ class FreqaiDataDrawer:
 
     def purge_old_models(self) -> None:
 
+        num_keep = self.freqai_info["purge_old_models"]
+        if not num_keep:
+            return
+        elif type(num_keep) == bool:
+            num_keep = 2
+
         model_folders = [x for x in self.full_path.iterdir() if x.is_dir()]
 
         pattern = re.compile(r"sub-train-(\w+)_(\d{10})")
@@ -404,11 +394,11 @@ class FreqaiDataDrawer:
                 delete_dict[coin]["timestamps"][int(timestamp)] = dir
 
         for coin in delete_dict:
-            if delete_dict[coin]["num_folders"] > 2:
+            if delete_dict[coin]["num_folders"] > num_keep:
                 sorted_dict = collections.OrderedDict(
                     sorted(delete_dict[coin]["timestamps"].items())
                 )
-                num_delete = len(sorted_dict) - 2
+                num_delete = len(sorted_dict) - num_keep
                 deleted = 0
                 for k, v in sorted_dict.items():
                     if deleted >= num_delete:
@@ -417,12 +407,6 @@ class FreqaiDataDrawer:
                     shutil.rmtree(v)
                     deleted += 1
 
-    def update_follower_metadata(self):
-        # follower needs to load from disk to get any changes made by leader to pair_dict
-        self.load_drawer_from_disk()
-        if self.config.get("freqai", {}).get("purge_old_models", False):
-            self.purge_old_models()
-
     def save_metadata(self, dk: FreqaiDataKitchen) -> None:
         """
         Saves only metadata for backtesting studies if user prefers
@@ -440,7 +424,7 @@ class FreqaiDataDrawer:
         dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns)
         dk.data["label_list"] = dk.label_list
 
-        with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp:
+        with (save_path / f"{dk.model_filename}_metadata.json").open("w") as fp:
             rapidjson.dump(dk.data, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE)
 
         return
@@ -473,7 +457,7 @@ class FreqaiDataDrawer:
         dk.data["training_features_list"] = dk.training_features_list
         dk.data["label_list"] = dk.label_list
         # store the metadata
-        with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp:
+        with (save_path / f"{dk.model_filename}_metadata.json").open("w") as fp:
             rapidjson.dump(dk.data, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE)
 
         # save the train data to file so we can check preds for area of applicability later
@@ -487,7 +471,7 @@ class FreqaiDataDrawer:
 
         if self.freqai_info["feature_parameters"].get("principal_component_analysis"):
             cloudpickle.dump(
-                dk.pca, open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "wb")
+                dk.pca, (dk.data_path / f"{dk.model_filename}_pca_object.pkl").open("wb")
             )
 
         self.model_dictionary[coin] = model
@@ -507,7 +491,7 @@ class FreqaiDataDrawer:
         Load only metadata into datakitchen to increase performance during
         presaved backtesting (prediction file loading).
         """
-        with open(dk.data_path / f"{dk.model_filename}_metadata.json", "r") as fp:
+        with (dk.data_path / f"{dk.model_filename}_metadata.json").open("r") as fp:
             dk.data = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
             dk.training_features_list = dk.data["training_features_list"]
             dk.label_list = dk.data["label_list"]
@@ -530,7 +514,7 @@ class FreqaiDataDrawer:
             dk.data = self.meta_data_dictionary[coin]["meta_data"]
             dk.data_dictionary["train_features"] = self.meta_data_dictionary[coin]["train_df"]
         else:
-            with open(dk.data_path / f"{dk.model_filename}_metadata.json", "r") as fp:
+            with (dk.data_path / f"{dk.model_filename}_metadata.json").open("r") as fp:
                 dk.data = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
 
             dk.data_dictionary["train_features"] = pd.read_pickle(
@@ -568,7 +552,7 @@ class FreqaiDataDrawer:
 
         if self.config["freqai"]["feature_parameters"]["principal_component_analysis"]:
             dk.pca = cloudpickle.load(
-                open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "rb")
+                (dk.data_path / f"{dk.model_filename}_pca_object.pkl").open("rb")
             )
 
         return model
@@ -586,12 +570,12 @@ class FreqaiDataDrawer:
 
             for pair in dk.all_pairs:
                 for tf in feat_params.get("include_timeframes"):
-
+                    hist_df = history_data[pair][tf]
                     # check if newest candle is already appended
                     df_dp = strategy.dp.get_pair_dataframe(pair, tf)
                     if len(df_dp.index) == 0:
                         continue
-                    if str(history_data[pair][tf].iloc[-1]["date"]) == str(
+                    if str(hist_df.iloc[-1]["date"]) == str(
                         df_dp.iloc[-1:]["date"].iloc[-1]
                     ):
                         continue
@@ -599,21 +583,30 @@ class FreqaiDataDrawer:
                     try:
                         index = (
                             df_dp.loc[
-                                df_dp["date"] == history_data[pair][tf].iloc[-1]["date"]
+                                df_dp["date"] == hist_df.iloc[-1]["date"]
                             ].index[0]
                             + 1
                         )
                     except IndexError:
-                        logger.warning(
+                        if hist_df.iloc[-1]['date'] < df_dp['date'].iloc[0]:
-                            f"Unable to update pair history for {pair}. "
+                            raise OperationalException("In memory historical data is older than "
-                            "If this does not resolve itself after 1 additional candle, "
+                                                       f"oldest DataProvider candle for {pair} on "
-                            "please report the error to #freqai discord channel"
+                                                       f"timeframe {tf}")
-                        )
+                        else:
-                        return
+                            index = -1
+                            logger.warning(
+                                f"No common dates in historical data and dataprovider for {pair}. "
+                                f"Appending latest dataprovider candle to historical data "
+                                "but please be aware that there is likely a gap in the historical "
+                                "data. \n"
+                                f"Historical data ends at {hist_df.iloc[-1]['date']} "
+                                f"while dataprovider starts at {df_dp['date'].iloc[0]} and"
+                                f"ends at {df_dp['date'].iloc[0]}."
+                            )
 
                     history_data[pair][tf] = pd.concat(
                         [
-                            history_data[pair][tf],
+                            hist_df,
                             df_dp.iloc[index:],
                         ],
                         ignore_index=True,

freqtrade/freqai/data_kitchen.py

@@ -1,6 +1,7 @@
 import copy
 import inspect
 import logging
+import random
 import shutil
 from datetime import datetime, timezone
 from math import cos, sin
@@ -170,6 +171,19 @@ class FreqaiDataKitchen:
             train_labels = labels
             train_weights = weights
 
+        if feat_dict["shuffle_after_split"]:
+            rint1 = random.randint(0, 100)
+            rint2 = random.randint(0, 100)
+            train_features = train_features.sample(
+                frac=1, random_state=rint1).reset_index(drop=True)
+            train_labels = train_labels.sample(frac=1, random_state=rint1).reset_index(drop=True)
+            train_weights = pd.DataFrame(train_weights).sample(
+                frac=1, random_state=rint1).reset_index(drop=True).to_numpy()[:, 0]
+            test_features = test_features.sample(frac=1, random_state=rint2).reset_index(drop=True)
+            test_labels = test_labels.sample(frac=1, random_state=rint2).reset_index(drop=True)
+            test_weights = pd.DataFrame(test_weights).sample(
+                frac=1, random_state=rint2).reset_index(drop=True).to_numpy()[:, 0]
+
         # Simplest way to reverse the order of training and test data:
         if self.freqai_config['feature_parameters'].get('reverse_train_test_order', False):
             return self.build_data_dictionary(
@@ -1301,123 +1315,54 @@ class FreqaiDataKitchen:
         dataframe: DataFrame = dataframe containing populated indicators
         """
 
-        # this is a hack to check if the user is using the populate_any_indicators function
+        # check if the user is using the deprecated populate_any_indicators function
         new_version = inspect.getsource(strategy.populate_any_indicators) == (
             inspect.getsource(IStrategy.populate_any_indicators))
 
-        if new_version:
+        if not new_version:
-            tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes")
+            raise OperationalException(
-            pairs: List[str] = self.freqai_config["feature_parameters"].get(
+                "You are using the `populate_any_indicators()` function"
-                "include_corr_pairlist", [])
+                " which was deprecated on March 1, 2023. Please refer "
+                "to the strategy migration guide to use the new "
+                "feature_engineering_* methods: \n"
+                "https://www.freqtrade.io/en/stable/strategy_migration/#freqai-strategy \n"
+                "And the feature_engineering_* documentation: \n"
+                "https://www.freqtrade.io/en/latest/freqai-feature-engineering/"
+                )
 
-            for tf in tfs:
-                if tf not in base_dataframes:
-                    base_dataframes[tf] = pd.DataFrame()
-                for p in pairs:
-                    if p not in corr_dataframes:
-                        corr_dataframes[p] = {}
-                    if tf not in corr_dataframes[p]:
-                        corr_dataframes[p][tf] = pd.DataFrame()
-
-            if not prediction_dataframe.empty:
-                dataframe = prediction_dataframe.copy()
-            else:
-                dataframe = base_dataframes[self.config["timeframe"]].copy()
-
-            corr_pairs: List[str] = self.freqai_config["feature_parameters"].get(
-                "include_corr_pairlist", [])
-            dataframe = self.populate_features(dataframe.copy(), pair, strategy,
-                                               corr_dataframes, base_dataframes)
-            metadata = {"pair": pair}
-            dataframe = strategy.feature_engineering_standard(dataframe.copy(), metadata=metadata)
-            # ensure corr pairs are always last
-            for corr_pair in corr_pairs:
-                if pair == corr_pair:
-                    continue  # dont repeat anything from whitelist
-                if corr_pairs and do_corr_pairs:
-                    dataframe = self.populate_features(dataframe.copy(), corr_pair, strategy,
-                                                       corr_dataframes, base_dataframes, True)
-
-            dataframe = strategy.set_freqai_targets(dataframe.copy(), metadata=metadata)
-
-            self.get_unique_classes_from_labels(dataframe)
-
-            dataframe = self.remove_special_chars_from_feature_names(dataframe)
-
-            if self.config.get('reduce_df_footprint', False):
-                dataframe = reduce_dataframe_footprint(dataframe)
-
-            return dataframe
-
-        else:
-            # the user is using the populate_any_indicators functions which is deprecated
-
-            df = self.use_strategy_to_populate_indicators_old_version(
-                strategy, corr_dataframes, base_dataframes, pair,
-                prediction_dataframe, do_corr_pairs)
-            return df
-
-    def use_strategy_to_populate_indicators_old_version(
-        self,
-        strategy: IStrategy,
-        corr_dataframes: dict = {},
-        base_dataframes: dict = {},
-        pair: str = "",
-        prediction_dataframe: DataFrame = pd.DataFrame(),
-        do_corr_pairs: bool = True,
-    ) -> DataFrame:
-        """
-        Use the user defined strategy for populating indicators during retrain
-        :param strategy: IStrategy = user defined strategy object
-        :param corr_dataframes: dict = dict containing the df pair dataframes
-                                (for user defined timeframes)
-        :param base_dataframes: dict = dict containing the current pair dataframes
-                                (for user defined timeframes)
-        :param metadata: dict = strategy furnished pair metadata
-        :return:
-        dataframe: DataFrame = dataframe containing populated indicators
-        """
-
-        # for prediction dataframe creation, we let dataprovider handle everything in the strategy
-        # so we create empty dictionaries, which allows us to pass None to
-        # `populate_any_indicators()`. Signaling we want the dp to give us the live dataframe.
         tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes")
-        pairs: List[str] = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
+        pairs: List[str] = self.freqai_config["feature_parameters"].get(
+            "include_corr_pairlist", [])
+
+        for tf in tfs:
+            if tf not in base_dataframes:
+                base_dataframes[tf] = pd.DataFrame()
+            for p in pairs:
+                if p not in corr_dataframes:
+                    corr_dataframes[p] = {}
+                if tf not in corr_dataframes[p]:
+                    corr_dataframes[p][tf] = pd.DataFrame()
+
         if not prediction_dataframe.empty:
             dataframe = prediction_dataframe.copy()
-            for tf in tfs:
-                base_dataframes[tf] = None
-                for p in pairs:
-                    if p not in corr_dataframes:
-                        corr_dataframes[p] = {}
-                    corr_dataframes[p][tf] = None
         else:
             dataframe = base_dataframes[self.config["timeframe"]].copy()
 
-        sgi = False
+        corr_pairs: List[str] = self.freqai_config["feature_parameters"].get(
-        for tf in tfs:
+            "include_corr_pairlist", [])
-            if tf == tfs[-1]:
+        dataframe = self.populate_features(dataframe.copy(), pair, strategy,
-                sgi = True  # doing this last allows user to use all tf raw prices in labels
+                                           corr_dataframes, base_dataframes)
-            dataframe = strategy.populate_any_indicators(
+        metadata = {"pair": pair}
-                pair,
+        dataframe = strategy.feature_engineering_standard(dataframe.copy(), metadata=metadata)
-                dataframe.copy(),
-                tf,
-                informative=base_dataframes[tf],
-                set_generalized_indicators=sgi
-            )
-
         # ensure corr pairs are always last
-        for corr_pair in pairs:
+        for corr_pair in corr_pairs:
             if pair == corr_pair:
                 continue  # dont repeat anything from whitelist
-            for tf in tfs:
+            if corr_pairs and do_corr_pairs:
-                if pairs and do_corr_pairs:
+                dataframe = self.populate_features(dataframe.copy(), corr_pair, strategy,
-                    dataframe = strategy.populate_any_indicators(
+                                                   corr_dataframes, base_dataframes, True)
-                        corr_pair,
+
-                        dataframe.copy(),
+        dataframe = strategy.set_freqai_targets(dataframe.copy(), metadata=metadata)
-                        tf,
-                        informative=corr_dataframes[corr_pair][tf]
-                    )
 
         self.get_unique_classes_from_labels(dataframe)
 
@@ -1548,3 +1493,25 @@ class FreqaiDataKitchen:
             dataframe.columns = dataframe.columns.str.replace(c, "")
 
         return dataframe
+
+    def buffer_timerange(self, timerange: TimeRange):
+        """
+        Buffer the start and end of the timerange. This is used *after* the indicators
+        are populated.
+
+        The main example use is when predicting maxima and minima, the argrelextrema
+        function  cannot know the maxima/minima at the edges of the timerange. To improve
+        model accuracy, it is best to compute argrelextrema on the full timerange
+        and then use this function to cut off the edges (buffer) by the kernel.
+
+        In another case, if the targets are set to a shifted price movement, this
+        buffer is unnecessary because the shifted candles at the end of the timerange
+        will be NaN and FreqAI will automatically cut those off of the training
+        dataset.
+        """
+        buffer = self.freqai_config["feature_parameters"]["buffer_train_data_candles"]
+        if buffer:
+            timerange.stopts -= buffer * timeframe_to_seconds(self.config["timeframe"])
+            timerange.startts += buffer * timeframe_to_seconds(self.config["timeframe"])
+
+        return timerange

freqtrade/freqai/freqai_interface.py

@@ -1,4 +1,3 @@
-import inspect
 import logging
 import threading
 import time
@@ -106,8 +105,6 @@ class IFreqaiModel(ABC):
         self.max_system_threads = max(int(psutil.cpu_count() * 2 - 2), 1)
         self.can_short = True  # overridden in start() with strategy.can_short
 
-        self.warned_deprecated_populate_any_indicators = False
-
         record_params(config, self.full_path)
 
     def __getstate__(self):
@@ -138,9 +135,6 @@ class IFreqaiModel(ABC):
         self.data_provider = strategy.dp
         self.can_short = strategy.can_short
 
-        # check if the strategy has deprecated populate_any_indicators function
-        self.check_deprecated_populate_any_indicators(strategy)
-
         if self.live:
             self.inference_timer('start')
             self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
@@ -227,7 +221,7 @@ class IFreqaiModel(ABC):
                 logger.warning(f'{pair} not in current whitelist, removing from train queue.')
                 continue
 
-            (_, trained_timestamp, _) = self.dd.get_pair_dict_info(pair)
+            (_, trained_timestamp) = self.dd.get_pair_dict_info(pair)
 
             dk = FreqaiDataKitchen(self.config, self.live, pair)
             (
@@ -285,7 +279,7 @@ class IFreqaiModel(ABC):
         # following tr_train. Both of these windows slide through the
         # entire backtest
         for tr_train, tr_backtest in zip(dk.training_timeranges, dk.backtesting_timeranges):
-            (_, _, _) = self.dd.get_pair_dict_info(pair)
+            (_, _) = self.dd.get_pair_dict_info(pair)
             train_it += 1
             total_trains = len(dk.backtesting_timeranges)
             self.training_timerange = tr_train
@@ -330,6 +324,8 @@ class IFreqaiModel(ABC):
                 dataframe_base_backtest = strategy.set_freqai_targets(
                     dataframe_base_backtest, metadata=metadata)
 
+                tr_train = dk.buffer_timerange(tr_train)
+
                 dataframe_train = dk.slice_dataframe(tr_train, dataframe_base_train)
                 dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe_base_backtest)
 
@@ -382,7 +378,7 @@ class IFreqaiModel(ABC):
         """
 
         # get the model metadata associated with the current pair
-        (_, trained_timestamp, return_null_array) = self.dd.get_pair_dict_info(metadata["pair"])
+        (_, trained_timestamp) = self.dd.get_pair_dict_info(metadata["pair"])
 
         # append the historic data once per round
         if self.dd.historic_data:
@@ -489,7 +485,7 @@ class IFreqaiModel(ABC):
                 "strategy is furnishing the same features as the pretrained"
                 "model. In case of --strategy-list, please be aware that FreqAI "
                 "requires all strategies to maintain identical "
-                "populate_any_indicator() functions"
+                "feature_engineering_* functions"
             )
 
     def data_cleaning_train(self, dk: FreqaiDataKitchen) -> None:
@@ -569,7 +565,7 @@ class IFreqaiModel(ABC):
             file_type = ".h5"
         elif 'stable_baselines' in self.dd.model_type or 'sb3_contrib' == self.dd.model_type:
             file_type = ".zip"
-        path_to_modelfile = Path(dk.data_path / f"{dk.model_filename}_model.{file_type}")
+        path_to_modelfile = Path(dk.data_path / f"{dk.model_filename}_model{file_type}")
         file_exists = path_to_modelfile.is_file()
         if file_exists:
             logger.info("Found model at %s", dk.data_path / dk.model_filename)
@@ -601,7 +597,7 @@ class IFreqaiModel(ABC):
         :param strategy: IStrategy = user defined strategy object
         :param dk: FreqaiDataKitchen = non-persistent data container for current coin/loop
         :param data_load_timerange: TimeRange = the amount of data to be loaded
-                                    for populate_any_indicators
+                                    for populating indicators
                                     (larger than new_trained_timerange so that
                                     new_trained_timerange does not contain any NaNs)
         """
@@ -614,6 +610,8 @@ class IFreqaiModel(ABC):
             strategy, corr_dataframes, base_dataframes, pair
         )
 
+        new_trained_timerange = dk.buffer_timerange(new_trained_timerange)
+
         unfiltered_dataframe = dk.slice_dataframe(new_trained_timerange, unfiltered_dataframe)
 
         # find the features indicated by strategy and store in datakitchen
@@ -629,8 +627,7 @@ class IFreqaiModel(ABC):
         if self.plot_features:
             plot_feature_importance(model, pair, dk, self.plot_features)
 
-        if self.freqai_info.get("purge_old_models", False):
+        self.dd.purge_old_models()
-            self.dd.purge_old_models()
 
     def set_initial_historic_predictions(
         self, pred_df: DataFrame, dk: FreqaiDataKitchen, pair: str, strat_df: DataFrame
@@ -806,7 +803,7 @@ class IFreqaiModel(ABC):
                 logger.warning("Couldn't cache corr_pair dataframes for improved performance. "
                                "Consider ensuring that the full coin/stake, e.g. XYZ/USD, "
                                "is included in the column names when you are creating features "
-                               "in `populate_any_indicators()`.")
+                               "in `feature_engineering_*` functions.")
             self.get_corr_dataframes = not bool(self.corr_dataframes)
         elif self.corr_dataframes:
             dataframe = dk.attach_corr_pair_columns(
@@ -933,26 +930,6 @@ class IFreqaiModel(ABC):
             dk.return_dataframe, saved_dataframe, how='left', left_on='date', right_on="date_pred")
         return dk
 
-    def check_deprecated_populate_any_indicators(self, strategy: IStrategy):
-        """
-        Check and warn if the deprecated populate_any_indicators function is used.
-        :param strategy: strategy object
-        """
-
-        if not self.warned_deprecated_populate_any_indicators:
-            self.warned_deprecated_populate_any_indicators = True
-            old_version = inspect.getsource(strategy.populate_any_indicators) != (
-                inspect.getsource(IStrategy.populate_any_indicators))
-
-            if old_version:
-                logger.warning("DEPRECATION WARNING: "
-                               "You are using the deprecated populate_any_indicators function. "
-                               "This function will raise an error on March 1 2023. "
-                               "Please update your strategy by using "
-                               "the new feature_engineering functions. See \n"
-                               "https://www.freqtrade.io/en/latest/freqai-feature-engineering/"
-                               "for details.")
-
     # Following methods which are overridden by user made prediction models.
     # See freqai/prediction_models/CatboostPredictionModel.py for an example.
 

freqtrade/freqai/prediction_models/ReinforcementLearner_multiproc.py

@@ -34,6 +34,11 @@ class ReinforcementLearner_multiproc(ReinforcementLearner):
         train_df = data_dictionary["train_features"]
         test_df = data_dictionary["test_features"]
 
+        if self.train_env:
+            self.train_env.close()
+        if self.eval_env:
+            self.eval_env.close()
+
         env_info = self.pack_env_dict(dk.pair)
 
         env_id = "train_env"

freqtrade/freqai/utils.py

@@ -211,7 +211,7 @@ def record_params(config: Dict[str, Any], full_path: Path) -> None:
         "pairs": config.get('exchange', {}).get('pair_whitelist')
     }
 
-    with open(params_record_path, "w") as handle:
+    with params_record_path.open("w") as handle:
         rapidjson.dump(
             run_params,
             handle,

freqtrade/freqtradebot.py

@@ -127,7 +127,7 @@ class FreqtradeBot(LoggingMixin):
                 for minutes in [0, 15, 30, 45]:
                     t = str(time(time_slot, minutes, 2))
                     self._schedule.every().day.at(t).do(update)
-        self.last_process = datetime(1970, 1, 1, tzinfo=timezone.utc)
+        self.last_process: Optional[datetime] = None
 
         self.strategy.ft_bot_start()
         # Initialize protections AFTER bot start - otherwise parameters are not loaded.
@@ -633,7 +633,7 @@ class FreqtradeBot(LoggingMixin):
                 return
 
             remaining = (trade.amount - amount) * current_exit_rate
-            if remaining < min_exit_stake:
+            if min_exit_stake and remaining < min_exit_stake:
                 logger.info(f"Remaining amount of {remaining} would be smaller "
                             f"than the minimum of {min_exit_stake}.")
                 return
@@ -841,7 +841,7 @@ class FreqtradeBot(LoggingMixin):
 
     def cancel_stoploss_on_exchange(self, trade: Trade) -> Trade:
         # First cancelling stoploss on exchange ...
-        if self.strategy.order_types.get('stoploss_on_exchange') and trade.stoploss_order_id:
+        if trade.stoploss_order_id:
             try:
                 logger.info(f"Canceling stoploss on exchange for {trade}")
                 co = self.exchange.cancel_stoploss_order_with_result(
@@ -1275,8 +1275,7 @@ class FreqtradeBot(LoggingMixin):
         if order['side'] == trade.entry_side:
             self.handle_cancel_enter(trade, order, reason)
         else:
-            canceled = self.handle_cancel_exit(
+            canceled = self.handle_cancel_exit(trade, order, reason)
-                trade, order, reason)
             canceled_count = trade.get_exit_order_count()
             max_timeouts = self.config.get('unfilledtimeout', {}).get('exit_timeout_count', 0)
             if canceled and max_timeouts > 0 and canceled_count >= max_timeouts:
@@ -1315,7 +1314,7 @@ class FreqtradeBot(LoggingMixin):
                                                          default_retval=order_obj.price)(
                 trade=trade, order=order_obj, pair=trade.pair,
                 current_time=datetime.now(timezone.utc), proposed_rate=proposed_rate,
-                current_order_rate=order_obj.price, entry_tag=trade.enter_tag,
+                current_order_rate=order_obj.safe_price, entry_tag=trade.enter_tag,
                 side=trade.entry_side)
 
             replacing = True
@@ -1331,7 +1330,8 @@ class FreqtradeBot(LoggingMixin):
                     # place new order only if new price is supplied
                     self.execute_entry(
                         pair=trade.pair,
-                        stake_amount=(order_obj.remaining * order_obj.price / trade.leverage),
+                        stake_amount=(
+                            order_obj.safe_remaining * order_obj.safe_price / trade.leverage),
                         price=adjusted_entry_price,
                         trade=trade,
                         is_short=trade.is_short,
@@ -1345,6 +1345,8 @@ class FreqtradeBot(LoggingMixin):
         """
 
         for trade in Trade.get_open_order_trades():
+            if not trade.open_order_id:
+                continue
             try:
                 order = self.exchange.fetch_order(trade.open_order_id, trade.pair)
             except (ExchangeError):
@@ -1369,6 +1371,9 @@ class FreqtradeBot(LoggingMixin):
         """
         was_trade_fully_canceled = False
         side = trade.entry_side.capitalize()
+        if not trade.open_order_id:
+            logger.warning(f"No open order for {trade}.")
+            return False
 
         # Cancelled orders may have the status of 'canceled' or 'closed'
         if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
@@ -1455,7 +1460,7 @@ class FreqtradeBot(LoggingMixin):
                     return False
 
             try:
-                co = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
+                co = self.exchange.cancel_order_with_result(order['id'], trade.pair,
                                                             trade.amount)
             except InvalidOrderException:
                 logger.exception(
@@ -1640,7 +1645,7 @@ class FreqtradeBot(LoggingMixin):
             profit = trade.calc_profit(rate=order_rate, amount=amount, open_rate=trade.open_rate)
             profit_ratio = trade.calc_profit_ratio(order_rate, amount, trade.open_rate)
         else:
-            order_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
+            order_rate = trade.safe_close_rate
             profit = trade.calc_profit(rate=order_rate) + (0.0 if fill else trade.realized_profit)
             profit_ratio = trade.calc_profit_ratio(order_rate)
             amount = trade.amount
@@ -1695,7 +1700,7 @@ class FreqtradeBot(LoggingMixin):
             raise DependencyException(
                 f"Order_obj not found for {order_id}. This should not have happened.")
 
-        profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
+        profit_rate: float = trade.safe_close_rate
         profit_trade = trade.calc_profit(rate=profit_rate)
         current_rate = self.exchange.get_rate(
             trade.pair, side='exit', is_short=trade.is_short, refresh=False)
@@ -1738,7 +1743,8 @@ class FreqtradeBot(LoggingMixin):
 #
 
     def update_trade_state(
-            self, trade: Trade, order_id: str, action_order: Optional[Dict[str, Any]] = None,
+            self, trade: Trade, order_id: Optional[str],
+            action_order: Optional[Dict[str, Any]] = None,
             stoploss_order: bool = False, send_msg: bool = True) -> bool:
         """
         Checks trades with open orders and updates the amount if necessary

freqtrade/leverage/__init__.py

@@ -1,2 +1 @@
-# flake8: noqa: F401
+from freqtrade.leverage.interest import interest  # noqa: F401
-from freqtrade.leverage.interest import interest

freqtrade/loggers.py

@@ -103,9 +103,9 @@ def setup_logging(config: Config) -> None:
             logging.root.addHandler(handler_sl)
         elif s[0] == 'journald':  # pragma: no cover
             try:
-                from systemd.journal import JournaldLogHandler
+                from cysystemd.journal import JournaldLogHandler
             except ImportError:
-                raise OperationalException("You need the systemd python package be installed in "
+                raise OperationalException("You need the cysystemd python package be installed in "
                                            "order to use logging to journald.")
             handler_jd = get_existing_handlers(JournaldLogHandler)
             if handler_jd:

freqtrade/misc.py

@@ -81,7 +81,7 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool =
     else:
         if log:
             logger.info(f'dumping json to "{filename}"')
-        with open(filename, 'w') as fp:
+        with filename.open('w') as fp:
             rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE)
 
     logger.debug(f'done json to "{filename}"')
@@ -98,7 +98,7 @@ def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None:
 
     if log:
         logger.info(f'dumping joblib to "{filename}"')
-    with open(filename, 'wb') as fp:
+    with filename.open('wb') as fp:
         joblib.dump(data, fp)
     logger.debug(f'done joblib dump to "{filename}"')
 
@@ -112,7 +112,7 @@ def json_load(datafile: IO) -> Any:
     return rapidjson.load(datafile, number_mode=rapidjson.NM_NATIVE)
 
 
-def file_load_json(file):
+def file_load_json(file: Path):
 
     if file.suffix != ".gz":
         gzipfile = file.with_suffix(file.suffix + '.gz')
@@ -125,7 +125,7 @@ def file_load_json(file):
             pairdata = json_load(datafile)
     elif file.is_file():
         logger.debug(f"Loading historical data from file {file}")
-        with open(file) as datafile:
+        with file.open() as datafile:
             pairdata = json_load(datafile)
     else:
         return None

freqtrade/mixins/__init__.py

@@ -1,2 +1 @@
-# flake8: noqa: F401
+from freqtrade.mixins.logging_mixin import LoggingMixin  # noqa: F401
-from freqtrade.mixins.logging_mixin import LoggingMixin

freqtrade/optimize/backtest_caching.py

@@ -29,7 +29,7 @@ def get_strategy_run_id(strategy) -> str:
     # Include _ft_params_from_file - so changing parameter files cause cache eviction
     digest.update(rapidjson.dumps(
         strategy._ft_params_from_file, default=str, number_mode=rapidjson.NM_NAN).encode('utf-8'))
-    with open(strategy.__file__, 'rb') as fp:
+    with Path(strategy.__file__).open('rb') as fp:
         digest.update(fp.read())
     return digest.hexdigest().lower()
 

freqtrade/optimize/backtesting.py

@@ -93,7 +93,7 @@ class Backtesting:
         if self.config.get('strategy_list'):
             if self.config.get('freqai', {}).get('enabled', False):
                 logger.warning("Using --strategy-list with FreqAI REQUIRES all strategies "
-                               "to have identical populate_any_indicators.")
+                               "to have identical feature_engineering_* functions.")
             for strat in list(self.config['strategy_list']):
                 stratconf = deepcopy(self.config)
                 stratconf['strategy'] = strat
@@ -440,7 +440,8 @@ class Backtesting:
                               side_1 * abs(self.strategy.trailing_stop_positive / leverage)))
             else:
                 # Worst case: price ticks tiny bit above open and dives down.
-                stop_rate = row[OPEN_IDX] * (1 - side_1 * abs(trade.stop_loss_pct / leverage))
+                stop_rate = row[OPEN_IDX] * (1 - side_1 * abs(
+                    (trade.stop_loss_pct or 0.0) / leverage))
                 if is_short:
                     assert stop_rate > row[LOW_IDX]
                 else:
@@ -472,7 +473,7 @@ class Backtesting:
             # - (Expected abs profit - open_rate - open_fee) / (fee_close -1)
             roi_rate = trade.open_rate * roi / leverage
             open_fee_rate = side_1 * trade.open_rate * (1 + side_1 * trade.fee_open)
-            close_rate = -(roi_rate + open_fee_rate) / (trade.fee_close - side_1 * 1)
+            close_rate = -(roi_rate + open_fee_rate) / ((trade.fee_close or 0.0) - side_1 * 1)
             if is_short:
                 is_new_roi = row[OPEN_IDX] < close_rate
             else:
@@ -563,7 +564,7 @@ class Backtesting:
             pos_trade = self._get_exit_for_signal(trade, row, exit_, amount)
             if pos_trade is not None:
                 order = pos_trade.orders[-1]
-                if self._get_order_filled(order.price, row):
+                if self._get_order_filled(order.ft_price, row):
                     order.close_bt_order(current_date, trade)
                     trade.recalc_trade_from_orders()
                 self.wallets.update()
@@ -664,6 +665,7 @@ class Backtesting:
             side=trade.exit_side,
             order_type=order_type,
             status="open",
+            ft_price=close_rate,
             price=close_rate,
             average=close_rate,
             amount=amount,
@@ -887,6 +889,7 @@ class Backtesting:
                 order_date=current_time,
                 order_filled_date=current_time,
                 order_update_date=current_time,
+                ft_price=propose_rate,
                 price=propose_rate,
                 average=propose_rate,
                 amount=amount,
@@ -895,7 +898,7 @@ class Backtesting:
                 cost=stake_amount + trade.fee_open,
             )
             trade.orders.append(order)
-            if pos_adjust and self._get_order_filled(order.price, row):
+            if pos_adjust and self._get_order_filled(order.ft_price, row):
                 order.close_bt_order(current_time, trade)
             else:
                 trade.open_order_id = str(self.order_id_counter)
@@ -1008,15 +1011,15 @@ class Backtesting:
         # only check on new candles for open entry orders
         if order.side == trade.entry_side and current_time > order.order_date_utc:
             requested_rate = strategy_safe_wrapper(self.strategy.adjust_entry_price,
-                                                   default_retval=order.price)(
+                                                   default_retval=order.ft_price)(
                 trade=trade,  # type: ignore[arg-type]
                 order=order, pair=trade.pair, current_time=current_time,
-                proposed_rate=row[OPEN_IDX], current_order_rate=order.price,
+                proposed_rate=row[OPEN_IDX], current_order_rate=order.ft_price,
                 entry_tag=trade.enter_tag, side=trade.trade_direction
             )  # default value is current order price
 
             # cancel existing order whenever a new rate is requested (or None)
-            if requested_rate == order.price:
+            if requested_rate == order.ft_price:
                 # assumption: there can't be multiple open entry orders at any given time
                 return False
             else:
@@ -1028,7 +1031,8 @@ class Backtesting:
             if requested_rate:
                 self._enter_trade(pair=trade.pair, row=row, trade=trade,
                                   requested_rate=requested_rate,
-                                  requested_stake=(order.remaining * order.price / trade.leverage),
+                                  requested_stake=(
+                                    order.safe_remaining * order.ft_price / trade.leverage),
                                   direction='short' if trade.is_short else 'long')
                 self.replaced_entry_orders += 1
             else:
@@ -1095,7 +1099,7 @@ class Backtesting:
         for trade in list(LocalTrade.bt_trades_open_pp[pair]):
             # 3. Process entry orders.
             order = trade.select_order(trade.entry_side, is_open=True)
-            if order and self._get_order_filled(order.price, row):
+            if order and self._get_order_filled(order.ft_price, row):
                 order.close_bt_order(current_time, trade)
                 trade.open_order_id = None
                 self.wallets.update()
@@ -1106,7 +1110,7 @@ class Backtesting:
 
                 # 5. Process exit orders.
             order = trade.select_order(trade.exit_side, is_open=True)
-            if order and self._get_order_filled(order.price, row):
+            if order and self._get_order_filled(order.ft_price, row):
                 order.close_bt_order(current_time, trade)
                 trade.open_order_id = None
                 sub_trade = order.safe_amount_after_fee != trade.amount
@@ -1115,7 +1119,7 @@ class Backtesting:
                     trade.recalc_trade_from_orders()
                 else:
                     trade.close_date = current_time
-                    trade.close(order.price, show_msg=False)
+                    trade.close(order.ft_price, show_msg=False)
 
                     # logger.debug(f"{pair} - Backtesting exit {trade}")
                     LocalTrade.close_bt_trade(trade)

freqtrade/optimize/space/__init__.py

@@ -1,4 +1,3 @@
-# flake8: noqa: F401
+from skopt.space import Categorical, Dimension, Integer, Real  # noqa: F401
-from skopt.space import Categorical, Dimension, Integer, Real
 
-from .decimalspace import SKDecimal
+from .decimalspace import SKDecimal  # noqa: F401

freqtrade/persistence/base.py

@@ -1,7 +1,9 @@
 
-from typing import Any
+from sqlalchemy.orm import DeclarativeBase, Session, scoped_session
-
-from sqlalchemy.orm import declarative_base
 
 
-_DECL_BASE: Any = declarative_base()
+SessionType = scoped_session[Session]
+
+
+class ModelBase(DeclarativeBase):
+    pass

freqtrade/persistence/models.py

@@ -2,6 +2,7 @@
 This module contains the class to persist trades into SQLite
 """
 import logging
+from typing import Any, Dict
 
 from sqlalchemy import create_engine, inspect
 from sqlalchemy.exc import NoSuchModuleError
@@ -9,7 +10,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker
 from sqlalchemy.pool import StaticPool
 
 from freqtrade.exceptions import OperationalException
-from freqtrade.persistence.base import _DECL_BASE
+from freqtrade.persistence.base import ModelBase
 from freqtrade.persistence.migrations import check_migrate
 from freqtrade.persistence.pairlock import PairLock
 from freqtrade.persistence.trade_model import Order, Trade
@@ -29,7 +30,7 @@ def init_db(db_url: str) -> None:
     :param db_url: Database to use
     :return: None
     """
-    kwargs = {}
+    kwargs: Dict[str, Any] = {}
 
     if db_url == 'sqlite:///':
         raise OperationalException(
@@ -54,10 +55,12 @@ def init_db(db_url: str) -> None:
     # Scoped sessions proxy requests to the appropriate thread-local session.
     # We should use the scoped_session object - not a seperately initialized version
     Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=False))
+    Order._session = Trade._session
+    PairLock._session = Trade._session
     Trade.query = Trade._session.query_property()
     Order.query = Trade._session.query_property()
     PairLock.query = Trade._session.query_property()
 
     previous_tables = inspect(engine).get_table_names()
-    _DECL_BASE.metadata.create_all(engine)
+    ModelBase.metadata.create_all(engine)
-    check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables)
+    check_migrate(engine, decl_base=ModelBase, previous_tables=previous_tables)

freqtrade/persistence/pairlock.py

@@ -1,33 +1,36 @@
 from datetime import datetime, timezone
-from typing import Any, Dict, Optional
+from typing import Any, ClassVar, Dict, Optional
 
-from sqlalchemy import Boolean, Column, DateTime, Integer, String, or_
+from sqlalchemy import String, or_
-from sqlalchemy.orm import Query
+from sqlalchemy.orm import Mapped, Query, mapped_column
+from sqlalchemy.orm.scoping import _QueryDescriptorType
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT
-from freqtrade.persistence.base import _DECL_BASE
+from freqtrade.persistence.base import ModelBase, SessionType
 
 
-class PairLock(_DECL_BASE):
+class PairLock(ModelBase):
     """
     Pair Locks database model.
     """
     __tablename__ = 'pairlocks'
+    query: ClassVar[_QueryDescriptorType]
+    _session: ClassVar[SessionType]
 
-    id = Column(Integer, primary_key=True)
+    id: Mapped[int] = mapped_column(primary_key=True)
 
-    pair = Column(String(25), nullable=False, index=True)
+    pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True)
     # lock direction - long, short or * (for both)
-    side = Column(String(25), nullable=False, default="*")
+    side: Mapped[str] = mapped_column(String(25), nullable=False, default="*")
-    reason = Column(String(255), nullable=True)
+    reason: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
     # Time the pair was locked (start time)
-    lock_time = Column(DateTime(), nullable=False)
+    lock_time: Mapped[datetime] = mapped_column(nullable=False)
     # Time until the pair is locked (end time)
-    lock_end_time = Column(DateTime(), nullable=False, index=True)
+    lock_end_time: Mapped[datetime] = mapped_column(nullable=False, index=True)
 
-    active = Column(Boolean, nullable=False, default=True, index=True)
+    active: Mapped[bool] = mapped_column(nullable=False, default=True, index=True)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT)
         lock_end_time = self.lock_end_time.strftime(DATETIME_PRINT_FORMAT)
         return (

freqtrade/persistence/pairlock_middleware.py

@@ -133,8 +133,8 @@ class PairLocks():
             PairLock.query.session.commit()
         else:
             # used in backtesting mode; don't show log messages for speed
-            locks = PairLocks.get_pair_locks(None)
+            locksb = PairLocks.get_pair_locks(None)
-            for lock in locks:
+            for lock in locksb:
                 if lock.reason == reason:
                     lock.active = False
 

freqtrade/persistence/trade_model.py

@@ -5,11 +5,11 @@ import logging
 from collections import defaultdict
 from datetime import datetime, timedelta, timezone
 from math import isclose
-from typing import Any, Dict, List, Optional
+from typing import Any, ClassVar, Dict, List, Optional, cast
 
-from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String,
+from sqlalchemy import Enum, Float, ForeignKey, Integer, String, UniqueConstraint, desc, func
-                        UniqueConstraint, desc, func)
+from sqlalchemy.orm import Mapped, Query, lazyload, mapped_column, relationship
-from sqlalchemy.orm import Query, lazyload, relationship
+from sqlalchemy.orm.scoping import _QueryDescriptorType
 
 from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES,
                                  BuySell, LongShort)
@@ -17,14 +17,14 @@ from freqtrade.enums import ExitType, TradingMode
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import amount_to_contract_precision, price_to_precision
 from freqtrade.leverage import interest
-from freqtrade.persistence.base import _DECL_BASE
+from freqtrade.persistence.base import ModelBase, SessionType
 from freqtrade.util import FtPrecise
 
 
 logger = logging.getLogger(__name__)
 
 
-class Order(_DECL_BASE):
+class Order(ModelBase):
     """
     Order database model
     Keeps a record of all orders placed on the exchange
@@ -36,41 +36,44 @@ class Order(_DECL_BASE):
     Mirrors CCXT Order structure
     """
     __tablename__ = 'orders'
+    query: ClassVar[_QueryDescriptorType]
+    _session: ClassVar[SessionType]
+
     # Uniqueness should be ensured over pair, order_id
     # its likely that order_id is unique per Pair on some exchanges.
     __table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),)
 
-    id = Column(Integer, primary_key=True)
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True)
+    ft_trade_id: Mapped[int] = mapped_column(Integer, ForeignKey('trades.id'), index=True)
 
-    trade = relationship("Trade", back_populates="orders")
+    trade: Mapped[List["Trade"]] = relationship("Trade", back_populates="orders")
 
     # order_side can only be 'buy', 'sell' or 'stoploss'
-    ft_order_side = Column(String(25), nullable=False)
+    ft_order_side: Mapped[str] = mapped_column(String(25), nullable=False)
-    ft_pair = Column(String(25), nullable=False)
+    ft_pair: Mapped[str] = mapped_column(String(25), nullable=False)
-    ft_is_open = Column(Boolean, nullable=False, default=True, index=True)
+    ft_is_open: Mapped[bool] = mapped_column(nullable=False, default=True, index=True)
-    ft_amount = Column(Float(), nullable=False)
+    ft_amount: Mapped[float] = mapped_column(Float(), nullable=False)
-    ft_price = Column(Float(), nullable=False)
+    ft_price: Mapped[float] = mapped_column(Float(), nullable=False)
 
-    order_id = Column(String(255), nullable=False, index=True)
+    order_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
-    status = Column(String(255), nullable=True)
+    status: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
-    symbol = Column(String(25), nullable=True)
+    symbol: Mapped[Optional[str]] = mapped_column(String(25), nullable=True)
-    order_type = Column(String(50), nullable=True)
+    # TODO: type: order_type type is Optional[str]
-    side = Column(String(25), nullable=True)
+    order_type: Mapped[str] = mapped_column(String(50), nullable=True)
-    price = Column(Float(), nullable=True)
+    side: Mapped[str] = mapped_column(String(25), nullable=True)
-    average = Column(Float(), nullable=True)
+    price: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-    amount = Column(Float(), nullable=True)
+    average: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-    filled = Column(Float(), nullable=True)
+    amount: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-    remaining = Column(Float(), nullable=True)
+    filled: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-    cost = Column(Float(), nullable=True)
+    remaining: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-    stop_price = Column(Float(), nullable=True)
+    cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-    order_date = Column(DateTime(), nullable=True, default=datetime.utcnow)
+    stop_price: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-    order_filled_date = Column(DateTime(), nullable=True)
+    order_date: Mapped[datetime] = mapped_column(nullable=True, default=datetime.utcnow)
-    order_update_date = Column(DateTime(), nullable=True)
+    order_filled_date: Mapped[Optional[datetime]] = mapped_column(nullable=True)
+    order_update_date: Mapped[Optional[datetime]] = mapped_column(nullable=True)
+    funding_fee: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
 
-    funding_fee = Column(Float(), nullable=True)
+    ft_fee_base: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-
-    ft_fee_base = Column(Float(), nullable=True)
 
     @property
     def order_date_utc(self) -> datetime:
@@ -96,6 +99,10 @@ class Order(_DECL_BASE):
     def safe_filled(self) -> float:
         return self.filled if self.filled is not None else self.amount or 0.0
 
+    @property
+    def safe_cost(self) -> float:
+        return self.cost or 0.0
+
     @property
     def safe_remaining(self) -> float:
         return (
@@ -151,7 +158,7 @@ class Order(_DECL_BASE):
         self.order_update_date = datetime.now(timezone.utc)
 
     def to_ccxt_object(self) -> Dict[str, Any]:
-        return {
+        order: Dict[str, Any] = {
             'id': self.order_id,
             'symbol': self.ft_pair,
             'price': self.price,
@@ -169,6 +176,9 @@ class Order(_DECL_BASE):
             'fee': None,
             'info': {},
         }
+        if self.ft_order_side == 'stoploss':
+            order['ft_order_type'] = 'stoploss'
+        return order
 
     def to_json(self, entry_side: str, minified: bool = False) -> Dict[str, Any]:
         resp = {
@@ -210,7 +220,7 @@ class Order(_DECL_BASE):
         # Assumes backtesting will use date_last_filled_utc to calculate future funding fees.
         self.funding_fee = trade.funding_fees
 
-        if (self.ft_order_side == trade.entry_side):
+        if (self.ft_order_side == trade.entry_side and self.price):
             trade.open_rate = self.price
             trade.recalc_trade_from_orders()
             trade.adjust_stop_loss(trade.open_rate, trade.stop_loss_pct, refresh=True)
@@ -290,15 +300,15 @@ class LocalTrade():
 
     exchange: str = ''
     pair: str = ''
-    base_currency: str = ''
+    base_currency: Optional[str] = ''
-    stake_currency: str = ''
+    stake_currency: Optional[str] = ''
     is_open: bool = True
     fee_open: float = 0.0
     fee_open_cost: Optional[float] = None
-    fee_open_currency: str = ''
+    fee_open_currency: Optional[str] = ''
-    fee_close: float = 0.0
+    fee_close: Optional[float] = 0.0
     fee_close_cost: Optional[float] = None
-    fee_close_currency: str = ''
+    fee_close_currency: Optional[str] = ''
     open_rate: float = 0.0
     open_rate_requested: Optional[float] = None
     # open_trade_value - calculated via _calc_open_trade_value
@@ -308,7 +318,7 @@ class LocalTrade():
     close_profit: Optional[float] = None
     close_profit_abs: Optional[float] = None
     stake_amount: float = 0.0
-    max_stake_amount: float = 0.0
+    max_stake_amount: Optional[float] = 0.0
     amount: float = 0.0
     amount_requested: Optional[float] = None
     open_date: datetime
@@ -317,9 +327,9 @@ class LocalTrade():
     # absolute value of the stop loss
     stop_loss: float = 0.0
     # percentage value of the stop loss
-    stop_loss_pct: float = 0.0
+    stop_loss_pct: Optional[float] = 0.0
     # absolute value of the initial stop loss
-    initial_stop_loss: float = 0.0
+    initial_stop_loss: Optional[float] = 0.0
     # percentage value of the initial stop loss
     initial_stop_loss_pct: Optional[float] = None
     # stoploss order id which is on exchange
@@ -327,12 +337,12 @@ class LocalTrade():
     # last update time of the stoploss order on exchange
     stoploss_last_update: Optional[datetime] = None
     # absolute value of the highest reached price
-    max_rate: float = 0.0
+    max_rate: Optional[float] = None
     # Lowest price reached
-    min_rate: float = 0.0
+    min_rate: Optional[float] = None
-    exit_reason: str = ''
+    exit_reason: Optional[str] = ''
-    exit_order_status: str = ''
+    exit_order_status: Optional[str] = ''
-    strategy: str = ''
+    strategy: Optional[str] = ''
     enter_tag: Optional[str] = None
     timeframe: Optional[int] = None
 
@@ -589,7 +599,7 @@ class LocalTrade():
 
         self.stop_loss_pct = -1 * abs(percent)
 
-    def adjust_stop_loss(self, current_price: float, stoploss: float,
+    def adjust_stop_loss(self, current_price: float, stoploss: Optional[float],
                          initial: bool = False, refresh: bool = False) -> None:
         """
         This adjusts the stop loss to it's most recently observed setting
@@ -598,7 +608,7 @@ class LocalTrade():
         :param initial: Called to initiate stop_loss.
             Skips everything if self.stop_loss is already set.
         """
-        if initial and not (self.stop_loss is None or self.stop_loss == 0):
+        if stoploss is None or (initial and not (self.stop_loss is None or self.stop_loss == 0)):
             # Don't modify if called with initial and nothing to do
             return
         refresh = True if refresh and self.nr_of_successful_entries == 1 else False
@@ -637,7 +647,7 @@ class LocalTrade():
             f"initial_stop_loss={self.initial_stop_loss:.8f}, "
             f"stop_loss={self.stop_loss:.8f}. "
             f"Trailing stoploss saved us: "
-            f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f}.")
+            f"{float(self.stop_loss) - float(self.initial_stop_loss or 0.0):.8f}.")
 
     def update_trade(self, order: Order) -> None:
         """
@@ -789,10 +799,10 @@ class LocalTrade():
 
         return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours)
 
-    def _calc_base_close(self, amount: FtPrecise, rate: float, fee: float) -> FtPrecise:
+    def _calc_base_close(self, amount: FtPrecise, rate: float, fee: Optional[float]) -> FtPrecise:
 
         close_trade = amount * FtPrecise(rate)
-        fees = close_trade * FtPrecise(fee)
+        fees = close_trade * FtPrecise(fee or 0.0)
 
         if self.is_short:
             return close_trade + fees
@@ -1056,10 +1066,14 @@ class LocalTrade():
         return len(self.select_filled_orders('sell'))
 
     @property
-    def sell_reason(self) -> str:
+    def sell_reason(self) -> Optional[str]:
         """ DEPRECATED! Please use exit_reason instead."""
         return self.exit_reason
 
+    @property
+    def safe_close_rate(self) -> float:
+        return self.close_rate or self.close_rate_requested or 0.0
+
     @staticmethod
     def get_trades_proxy(*, pair: Optional[str] = None, is_open: Optional[bool] = None,
                          open_date: Optional[datetime] = None,
@@ -1121,7 +1135,7 @@ class LocalTrade():
     @staticmethod
     def get_open_trades() -> List[Any]:
         """
-        Query trades from persistence layer
+        Retrieve open trades
         """
         return Trade.get_trades_proxy(is_open=True)
 
@@ -1156,7 +1170,7 @@ class LocalTrade():
                 logger.info(f"New stoploss: {trade.stop_loss}.")
 
 
-class Trade(_DECL_BASE, LocalTrade):
+class Trade(ModelBase, LocalTrade):
     """
     Trade database model.
     Also handles updating and querying trades
@@ -1164,79 +1178,98 @@ class Trade(_DECL_BASE, LocalTrade):
     Note: Fields must be aligned with LocalTrade class
     """
     __tablename__ = 'trades'
+    query: ClassVar[_QueryDescriptorType]
+    _session: ClassVar[SessionType]
 
     use_db: bool = True
 
-    id = Column(Integer, primary_key=True)
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)  # type: ignore
 
-    orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan",
+    orders: Mapped[List[Order]] = relationship(
-                          lazy="selectin", innerjoin=True)
+        "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin",
+        innerjoin=True)  # type: ignore
 
-    exchange = Column(String(25), nullable=False)
+    exchange: Mapped[str] = mapped_column(String(25), nullable=False)  # type: ignore
-    pair = Column(String(25), nullable=False, index=True)
+    pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True)  # type: ignore
-    base_currency = Column(String(25), nullable=True)
+    base_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True)  # type: ignore
-    stake_currency = Column(String(25), nullable=True)
+    stake_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True)  # type: ignore
-    is_open = Column(Boolean, nullable=False, default=True, index=True)
+    is_open: Mapped[bool] = mapped_column(nullable=False, default=True, index=True)  # type: ignore
-    fee_open = Column(Float(), nullable=False, default=0.0)
+    fee_open: Mapped[float] = mapped_column(Float(), nullable=False, default=0.0)  # type: ignore
-    fee_open_cost = Column(Float(), nullable=True)
+    fee_open_cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)  # type: ignore
-    fee_open_currency = Column(String(25), nullable=True)
+    fee_open_currency: Mapped[Optional[str]] = mapped_column(
-    fee_close = Column(Float(), nullable=False, default=0.0)
+        String(25), nullable=True)  # type: ignore
-    fee_close_cost = Column(Float(), nullable=True)
+    fee_close: Mapped[Optional[float]] = mapped_column(
-    fee_close_currency = Column(String(25), nullable=True)
+        Float(), nullable=False, default=0.0)  # type: ignore
-    open_rate: float = Column(Float())
+    fee_close_cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)  # type: ignore
-    open_rate_requested = Column(Float())
+    fee_close_currency: Mapped[Optional[str]] = mapped_column(
+        String(25), nullable=True)  # type: ignore
+    open_rate: Mapped[float] = mapped_column(Float())  # type: ignore
+    open_rate_requested: Mapped[Optional[float]] = mapped_column(
+        Float(), nullable=True)  # type: ignore
     # open_trade_value - calculated via _calc_open_trade_value
-    open_trade_value = Column(Float())
+    open_trade_value: Mapped[float] = mapped_column(Float(), nullable=True)  # type: ignore
-    close_rate: Optional[float] = Column(Float())
+    close_rate: Mapped[Optional[float]] = mapped_column(Float())  # type: ignore
-    close_rate_requested = Column(Float())
+    close_rate_requested: Mapped[Optional[float]] = mapped_column(Float())  # type: ignore
-    realized_profit = Column(Float(), default=0.0)
+    realized_profit: Mapped[float] = mapped_column(
-    close_profit = Column(Float())
+        Float(), default=0.0, nullable=True)  # type: ignore
-    close_profit_abs = Column(Float())
+    close_profit: Mapped[Optional[float]] = mapped_column(Float())  # type: ignore
-    stake_amount = Column(Float(), nullable=False)
+    close_profit_abs: Mapped[Optional[float]] = mapped_column(Float())  # type: ignore
-    max_stake_amount = Column(Float())
+    stake_amount: Mapped[float] = mapped_column(Float(), nullable=False)  # type: ignore
-    amount = Column(Float())
+    max_stake_amount: Mapped[Optional[float]] = mapped_column(Float())  # type: ignore
-    amount_requested = Column(Float())
+    amount: Mapped[float] = mapped_column(Float())  # type: ignore
-    open_date = Column(DateTime(), nullable=False, default=datetime.utcnow)
+    amount_requested: Mapped[Optional[float]] = mapped_column(Float())  # type: ignore
-    close_date = Column(DateTime())
+    open_date: Mapped[datetime] = mapped_column(
-    open_order_id = Column(String(255))
+        nullable=False, default=datetime.utcnow)  # type: ignore
+    close_date: Mapped[Optional[datetime]] = mapped_column()  # type: ignore
+    open_order_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)  # type: ignore
     # absolute value of the stop loss
-    stop_loss = Column(Float(), nullable=True, default=0.0)
+    stop_loss: Mapped[float] = mapped_column(Float(), nullable=True, default=0.0)  # type: ignore
     # percentage value of the stop loss
-    stop_loss_pct = Column(Float(), nullable=True)
+    stop_loss_pct: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)  # type: ignore
     # absolute value of the initial stop loss
-    initial_stop_loss = Column(Float(), nullable=True, default=0.0)
+    initial_stop_loss: Mapped[Optional[float]] = mapped_column(
+        Float(), nullable=True, default=0.0)  # type: ignore
     # percentage value of the initial stop loss
-    initial_stop_loss_pct = Column(Float(), nullable=True)
+    initial_stop_loss_pct: Mapped[Optional[float]] = mapped_column(
+        Float(), nullable=True)  # type: ignore
     # stoploss order id which is on exchange
-    stoploss_order_id = Column(String(255), nullable=True, index=True)
+    stoploss_order_id: Mapped[Optional[str]] = mapped_column(
+        String(255), nullable=True, index=True)  # type: ignore
     # last update time of the stoploss order on exchange
-    stoploss_last_update = Column(DateTime(), nullable=True)
+    stoploss_last_update: Mapped[Optional[datetime]] = mapped_column(nullable=True)  # type: ignore
     # absolute value of the highest reached price
-    max_rate = Column(Float(), nullable=True, default=0.0)
+    max_rate: Mapped[Optional[float]] = mapped_column(
+        Float(), nullable=True, default=0.0)  # type: ignore
     # Lowest price reached
-    min_rate = Column(Float(), nullable=True)
+    min_rate: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)  # type: ignore
-    exit_reason = Column(String(100), nullable=True)
+    exit_reason: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)  # type: ignore
-    exit_order_status = Column(String(100), nullable=True)
+    exit_order_status: Mapped[Optional[str]] = mapped_column(
-    strategy = Column(String(100), nullable=True)
+        String(100), nullable=True)  # type: ignore
-    enter_tag = Column(String(100), nullable=True)
+    strategy: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)  # type: ignore
-    timeframe = Column(Integer, nullable=True)
+    enter_tag: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)  # type: ignore
+    timeframe: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)  # type: ignore
 
-    trading_mode = Column(Enum(TradingMode), nullable=True)
+    trading_mode: Mapped[TradingMode] = mapped_column(
-    amount_precision = Column(Float(), nullable=True)
+        Enum(TradingMode), nullable=True)  # type: ignore
-    price_precision = Column(Float(), nullable=True)
+    amount_precision: Mapped[Optional[float]] = mapped_column(
-    precision_mode = Column(Integer, nullable=True)
+        Float(), nullable=True)  # type: ignore
-    contract_size = Column(Float(), nullable=True)
+    price_precision: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)  # type: ignore
+    precision_mode: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)  # type: ignore
+    contract_size: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)  # type: ignore
 
     # Leverage trading properties
-    leverage = Column(Float(), nullable=True, default=1.0)
+    leverage: Mapped[float] = mapped_column(Float(), nullable=True, default=1.0)  # type: ignore
-    is_short = Column(Boolean, nullable=False, default=False)
+    is_short: Mapped[bool] = mapped_column(nullable=False, default=False)  # type: ignore
-    liquidation_price = Column(Float(), nullable=True)
+    liquidation_price: Mapped[Optional[float]] = mapped_column(
+        Float(), nullable=True)  # type: ignore
 
     # Margin Trading Properties
-    interest_rate = Column(Float(), nullable=False, default=0.0)
+    interest_rate: Mapped[float] = mapped_column(
+        Float(), nullable=False, default=0.0)  # type: ignore
 
     # Futures properties
-    funding_fees = Column(Float(), nullable=True, default=None)
+    funding_fees: Mapped[Optional[float]] = mapped_column(
+        Float(), nullable=True, default=None)  # type: ignore
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -1282,7 +1315,7 @@ class Trade(_DECL_BASE, LocalTrade):
                 trade_filter.append(Trade.close_date > close_date)
             if is_open is not None:
                 trade_filter.append(Trade.is_open.is_(is_open))
-            return Trade.get_trades(trade_filter).all()
+            return cast(List[LocalTrade], Trade.get_trades(trade_filter).all())
         else:
             return LocalTrade.get_trades_proxy(
                 pair=pair, is_open=is_open,
@@ -1291,7 +1324,7 @@ class Trade(_DECL_BASE, LocalTrade):
             )
 
     @staticmethod
-    def get_trades(trade_filter=None, include_orders: bool = True) -> Query:
+    def get_trades(trade_filter=None, include_orders: bool = True) -> Query['Trade']:
         """
         Helper function to query Trades using filters.
         NOTE: Not supported in Backtesting.
@@ -1378,7 +1411,7 @@ class Trade(_DECL_BASE, LocalTrade):
         Returns List of dicts containing all Trades, including profit and trade count
         NOTE: Not supported in Backtesting.
         """
-        filters = [Trade.is_open.is_(False)]
+        filters: List = [Trade.is_open.is_(False)]
         if minutes:
             start_date = datetime.now(timezone.utc) - timedelta(minutes=minutes)
             filters.append(Trade.close_date >= start_date)
@@ -1411,7 +1444,7 @@ class Trade(_DECL_BASE, LocalTrade):
         NOTE: Not supported in Backtesting.
         """
 
-        filters = [Trade.is_open.is_(False)]
+        filters: List = [Trade.is_open.is_(False)]
         if (pair is not None):
             filters.append(Trade.pair == pair)
 
@@ -1444,7 +1477,7 @@ class Trade(_DECL_BASE, LocalTrade):
         NOTE: Not supported in Backtesting.
         """
 
-        filters = [Trade.is_open.is_(False)]
+        filters: List = [Trade.is_open.is_(False)]
         if (pair is not None):
             filters.append(Trade.pair == pair)
 
@@ -1477,7 +1510,7 @@ class Trade(_DECL_BASE, LocalTrade):
         NOTE: Not supported in Backtesting.
         """
 
-        filters = [Trade.is_open.is_(False)]
+        filters: List = [Trade.is_open.is_(False)]
         if (pair is not None):
             filters.append(Trade.pair == pair)
 

freqtrade/plugins/pairlist/RemotePairList.py

@@ -157,7 +157,7 @@ class RemotePairList(IPairList):
                 file_path = Path(filename)
 
                 if file_path.exists():
-                    with open(filename) as json_file:
+                    with file_path.open() as json_file:
                         # Load the JSON data into a dictionary
                         jsonparse = json.load(json_file)
 

freqtrade/rpc/__init__.py

@@ -1,3 +1,2 @@
-# flake8: noqa: F401
+from .rpc import RPC, RPCException, RPCHandler  # noqa: F401
-from .rpc import RPC, RPCException, RPCHandler
+from .rpc_manager import RPCManager  # noqa: F401
-from .rpc_manager import RPCManager

freqtrade/rpc/api_server/__init__.py

@@ -1,2 +1 @@
-# flake8: noqa: F401
+from .webserver import ApiServer  # noqa: F401
-from .webserver import ApiServer

freqtrade/rpc/api_server/api_backtest.py

@@ -10,7 +10,7 @@ from fastapi.exceptions import HTTPException
 from freqtrade.configuration.config_validation import validate_config_consistency
 from freqtrade.data.btanalysis import get_backtest_resultlist, load_and_merge_backtest_result
 from freqtrade.enums import BacktestState
-from freqtrade.exceptions import DependencyException
+from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.misc import deep_merge_dicts
 from freqtrade.rpc.api_server.api_schemas import (BacktestHistoryEntry, BacktestRequest,
                                                   BacktestResponse)
@@ -26,9 +26,10 @@ router = APIRouter()
 
 
 @router.post('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
-# flake8: noqa: C901
+async def api_start_backtest(  # noqa: C901
-async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: BackgroundTasks,
+        bt_settings: BacktestRequest, background_tasks: BackgroundTasks,
-                             config=Depends(get_config), ws_mode=Depends(is_webserver_mode)):
+        config=Depends(get_config), ws_mode=Depends(is_webserver_mode)):
+    ApiServer._bt['bt_error'] = None
     """Start backtesting if not done so already"""
     if ApiServer._bgtask_running:
         raise RPCException('Bot Background task already running')
@@ -60,30 +61,31 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
         asyncio.set_event_loop(asyncio.new_event_loop())
         try:
             # Reload strategy
-            lastconfig = ApiServer._bt_last_config
+            lastconfig = ApiServer._bt['last_config']
             strat = StrategyResolver.load_strategy(btconfig)
             validate_config_consistency(btconfig)
 
             if (
-                not ApiServer._bt
+                not ApiServer._bt['bt']
                 or lastconfig.get('timeframe') != strat.timeframe
                 or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail')
                 or lastconfig.get('timerange') != btconfig['timerange']
             ):
                 from freqtrade.optimize.backtesting import Backtesting
-                ApiServer._bt = Backtesting(btconfig)
+                ApiServer._bt['bt'] = Backtesting(btconfig)
-                ApiServer._bt.load_bt_data_detail()
+                ApiServer._bt['bt'].load_bt_data_detail()
             else:
-                ApiServer._bt.config = btconfig
+                ApiServer._bt['bt'].config = btconfig
-                ApiServer._bt.init_backtest()
+                ApiServer._bt['bt'].init_backtest()
             # Only reload data if timeframe changed.
             if (
-                not ApiServer._bt_data
+                not ApiServer._bt['data']
-                or not ApiServer._bt_timerange
+                or not ApiServer._bt['timerange']
                 or lastconfig.get('timeframe') != strat.timeframe
                 or lastconfig.get('timerange') != btconfig['timerange']
             ):
-                ApiServer._bt_data, ApiServer._bt_timerange = ApiServer._bt.load_bt_data()
+                ApiServer._bt['data'], ApiServer._bt['timerange'] = ApiServer._bt[
+                    'bt'].load_bt_data()
 
             lastconfig['timerange'] = btconfig['timerange']
             lastconfig['timeframe'] = strat.timeframe
@@ -91,34 +93,35 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
             lastconfig['enable_protections'] = btconfig.get('enable_protections')
             lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet')
 
-            ApiServer._bt.enable_protections = btconfig.get('enable_protections', False)
+            ApiServer._bt['bt'].enable_protections = btconfig.get('enable_protections', False)
-            ApiServer._bt.strategylist = [strat]
+            ApiServer._bt['bt'].strategylist = [strat]
-            ApiServer._bt.results = {}
+            ApiServer._bt['bt'].results = {}
-            ApiServer._bt.load_prior_backtest()
+            ApiServer._bt['bt'].load_prior_backtest()
 
-            ApiServer._bt.abort = False
+            ApiServer._bt['bt'].abort = False
-            if (ApiServer._bt.results and
+            if (ApiServer._bt['bt'].results and
-                    strat.get_strategy_name() in ApiServer._bt.results['strategy']):
+                    strat.get_strategy_name() in ApiServer._bt['bt'].results['strategy']):
                 # When previous result hash matches - reuse that result and skip backtesting.
                 logger.info(f'Reusing result of previous backtest for {strat.get_strategy_name()}')
             else:
-                min_date, max_date = ApiServer._bt.backtest_one_strategy(
+                min_date, max_date = ApiServer._bt['bt'].backtest_one_strategy(
-                    strat, ApiServer._bt_data, ApiServer._bt_timerange)
+                    strat, ApiServer._bt['data'], ApiServer._bt['timerange'])
 
-                ApiServer._bt.results = generate_backtest_stats(
+                ApiServer._bt['bt'].results = generate_backtest_stats(
-                    ApiServer._bt_data, ApiServer._bt.all_results,
+                    ApiServer._bt['data'], ApiServer._bt['bt'].all_results,
                     min_date=min_date, max_date=max_date)
 
             if btconfig.get('export', 'none') == 'trades':
                 store_backtest_stats(
-                    btconfig['exportfilename'], ApiServer._bt.results,
+                    btconfig['exportfilename'], ApiServer._bt['bt'].results,
                     datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
                     )
 
             logger.info("Backtest finished.")
 
-        except DependencyException as e:
+        except (Exception, OperationalException, DependencyException) as e:
-            logger.info(f"Backtesting caused an error: {e}")
+            logger.exception(f"Backtesting caused an error: {e}")
+            ApiServer._bt['bt_error'] = str(e)
             pass
         finally:
             ApiServer._bgtask_running = False
@@ -146,13 +149,14 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)):
         return {
             "status": "running",
             "running": True,
-            "step": ApiServer._bt.progress.action if ApiServer._bt else str(BacktestState.STARTUP),
+            "step": (ApiServer._bt['bt'].progress.action if ApiServer._bt['bt']
-            "progress": ApiServer._bt.progress.progress if ApiServer._bt else 0,
+                     else str(BacktestState.STARTUP)),
+            "progress": ApiServer._bt['bt'].progress.progress if ApiServer._bt['bt'] else 0,
             "trade_count": len(LocalTrade.trades),
             "status_msg": "Backtest running",
         }
 
-    if not ApiServer._bt:
+    if not ApiServer._bt['bt']:
         return {
             "status": "not_started",
             "running": False,
@@ -160,6 +164,14 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)):
             "progress": 0,
             "status_msg": "Backtest not yet executed"
         }
+    if ApiServer._bt['bt_error']:
+        return {
+            "status": "error",
+            "running": False,
+            "step": "",
+            "progress": 0,
+            "status_msg": f"Backtest failed with {ApiServer._bt['bt_error']}"
+        }
 
     return {
         "status": "ended",
@@ -167,7 +179,7 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)):
         "status_msg": "Backtest ended",
         "step": "finished",
         "progress": 1,
-        "backtest_result": ApiServer._bt.results,
+        "backtest_result": ApiServer._bt['bt'].results,
     }
 
 
@@ -182,12 +194,12 @@ def api_delete_backtest(ws_mode=Depends(is_webserver_mode)):
             "progress": 0,
             "status_msg": "Backtest running",
         }
-    if ApiServer._bt:
+    if ApiServer._bt['bt']:
-        ApiServer._bt.cleanup()
+        ApiServer._bt['bt'].cleanup()
-        del ApiServer._bt
+        del ApiServer._bt['bt']
-        ApiServer._bt = None
+        ApiServer._bt['bt'] = None
-        del ApiServer._bt_data
+        del ApiServer._bt['data']
-        ApiServer._bt_data = None
+        ApiServer._bt['data'] = None
         logger.info("Backtesting reset")
     return {
         "status": "reset",
@@ -208,7 +220,7 @@ def api_backtest_abort(ws_mode=Depends(is_webserver_mode)):
             "progress": 0,
             "status_msg": "Backtest ended",
         }
-    ApiServer._bt.abort = True
+    ApiServer._bt['bt'].abort = True
     return {
         "status": "stopping",
         "running": False,
@@ -218,14 +230,17 @@ def api_backtest_abort(ws_mode=Depends(is_webserver_mode)):
     }
 
 
-@router.get('/backtest/history', response_model=List[BacktestHistoryEntry], tags=['webserver', 'backtest'])
+@router.get('/backtest/history', response_model=List[BacktestHistoryEntry],
+            tags=['webserver', 'backtest'])
 def api_backtest_history(config=Depends(get_config), ws_mode=Depends(is_webserver_mode)):
     # Get backtest result history, read from metadata files
     return get_backtest_resultlist(config['user_data_dir'] / 'backtest_results')
 
 
-@router.get('/backtest/history/result', response_model=BacktestResponse, tags=['webserver', 'backtest'])
+@router.get('/backtest/history/result', response_model=BacktestResponse,
-def api_backtest_history_result(filename: str, strategy: str, config=Depends(get_config), ws_mode=Depends(is_webserver_mode)):
+            tags=['webserver', 'backtest'])
+def api_backtest_history_result(filename: str, strategy: str, config=Depends(get_config),
+                                ws_mode=Depends(is_webserver_mode)):
     # Get backtest result history, read from metadata files
     fn = config['user_data_dir'] / 'backtest_results' / filename
     results: Dict[str, Any] = {

freqtrade/rpc/api_server/api_schemas.py

@@ -228,24 +228,32 @@ class TradeSchema(BaseModel):
     fee_close: Optional[float]
     fee_close_cost: Optional[float]
     fee_close_currency: Optional[str]
+
     open_date: str
     open_timestamp: int
     open_rate: float
     open_rate_requested: Optional[float]
     open_trade_value: float
+
     close_date: Optional[str]
     close_timestamp: Optional[int]
     close_rate: Optional[float]
     close_rate_requested: Optional[float]
+
     close_profit: Optional[float]
     close_profit_pct: Optional[float]
     close_profit_abs: Optional[float]
+
     profit_ratio: Optional[float]
     profit_pct: Optional[float]
     profit_abs: Optional[float]
     profit_fiat: Optional[float]
+
+    realized_profit: float
+
     exit_reason: Optional[str]
     exit_order_status: Optional[str]
+
     stop_loss_abs: Optional[float]
     stop_loss_ratio: Optional[float]
     stop_loss_pct: Optional[float]
@@ -255,6 +263,7 @@ class TradeSchema(BaseModel):
     initial_stop_loss_abs: Optional[float]
     initial_stop_loss_ratio: Optional[float]
     initial_stop_loss_pct: Optional[float]
+
     min_rate: Optional[float]
     max_rate: Optional[float]
     open_order_id: Optional[str]
@@ -273,10 +282,10 @@ class OpenTradeSchema(TradeSchema):
     stoploss_current_dist_ratio: Optional[float]
     stoploss_entry_dist: Optional[float]
     stoploss_entry_dist_ratio: Optional[float]
-    current_profit: float
-    current_profit_abs: float
-    current_profit_pct: float
     current_rate: float
+    total_profit_abs: float
+    total_profit_fiat: Optional[float]
+
     open_order: Optional[str]
 
 
@@ -456,5 +465,5 @@ class SysInfo(BaseModel):
 
 
 class Health(BaseModel):
-    last_process: datetime
+    last_process: Optional[datetime]
-    last_process_ts: int
+    last_process_ts: Optional[int]

freqtrade/rpc/api_server/api_v1.py

@@ -346,4 +346,4 @@ def sysinfo():
 
 @router.get('/health', response_model=Health, tags=['info'])
 def health(rpc: RPC = Depends(get_rpc)):
-    return rpc._health()
+    return rpc.health()

freqtrade/rpc/api_server/webserver.py

@@ -36,10 +36,13 @@ class ApiServer(RPCHandler):
 
     _rpc: RPC
     # Backtesting type: Backtesting
-    _bt = None
+    _bt: Dict[str, Any] = {
-    _bt_data = None
+        'bt': None,
-    _bt_timerange = None
+        'data': None,
-    _bt_last_config: Config = {}
+        'timerange': None,
+        'last_config': {},
+        'bt_error': None,
+    }
     _has_rpc: bool = False
     _bgtask_running: bool = False
     _config: Config = {}

freqtrade/rpc/api_server/ws/__init__.py

@@ -1,7 +1,6 @@
-# flake8: noqa: F401
 # isort: off
-from freqtrade.rpc.api_server.ws.types import WebSocketType
+from freqtrade.rpc.api_server.ws.types import WebSocketType  # noqa: F401
-from freqtrade.rpc.api_server.ws.proxy import WebSocketProxy
+from freqtrade.rpc.api_server.ws.proxy import WebSocketProxy  # noqa: F401
-from freqtrade.rpc.api_server.ws.serializer import HybridJSONWebSocketSerializer
+from freqtrade.rpc.api_server.ws.serializer import HybridJSONWebSocketSerializer  # noqa: F401
-from freqtrade.rpc.api_server.ws.channel import WebSocketChannel
+from freqtrade.rpc.api_server.ws.channel import WebSocketChannel  # noqa: F401
-from freqtrade.rpc.api_server.ws.message_stream import MessageStream
+from freqtrade.rpc.api_server.ws.message_stream import MessageStream  # noqa: F401

freqtrade/rpc/rpc.py

@@ -19,8 +19,8 @@ from freqtrade.configuration.timerange import TimeRange
 from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT, Config
 from freqtrade.data.history import load_data
 from freqtrade.data.metrics import calculate_max_drawdown
-from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, SignalDirection, State,
+from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, MarketDirection, SignalDirection,
-                             TradingMode)
+                             State, TradingMode)
 from freqtrade.exceptions import ExchangeError, PricingError
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
 from freqtrade.loggers import bufferHandler
@@ -169,6 +169,7 @@ class RPC:
             for trade in trades:
                 order: Optional[Order] = None
                 current_profit_fiat: Optional[float] = None
+                total_profit_fiat: Optional[float] = None
                 if trade.open_order_id:
                     order = trade.select_order_by_order_id(trade.open_order_id)
                 # calculate profit and send message to user
@@ -188,8 +189,9 @@ class RPC:
                 else:
                     # Closed trade ...
                     current_rate = trade.close_rate
-                    current_profit = trade.close_profit
+                    current_profit = trade.close_profit or 0.0
-                    current_profit_abs = trade.close_profit_abs
+                    current_profit_abs = trade.close_profit_abs or 0.0
+                total_profit_abs = trade.realized_profit + current_profit_abs
 
                 # Calculate fiat profit
                 if not isnan(current_profit_abs) and self._fiat_converter:
@@ -198,6 +200,11 @@ class RPC:
                         self._freqtrade.config['stake_currency'],
                         self._freqtrade.config['fiat_display_currency']
                     )
+                    total_profit_fiat = self._fiat_converter.convert_amount(
+                        total_profit_abs,
+                        self._freqtrade.config['stake_currency'],
+                        self._freqtrade.config['fiat_display_currency']
+                    )
 
                 # Calculate guaranteed profit (in case of trailing stop)
                 stoploss_entry_dist = trade.calc_profit(trade.stop_loss)
@@ -210,14 +217,13 @@ class RPC:
                 trade_dict.update(dict(
                     close_profit=trade.close_profit if not trade.is_open else None,
                     current_rate=current_rate,
-                    current_profit=current_profit,  # Deprecated
-                    current_profit_pct=round(current_profit * 100, 2),  # Deprecated
-                    current_profit_abs=current_profit_abs,  # Deprecated
                     profit_ratio=current_profit,
                     profit_pct=round(current_profit * 100, 2),
                     profit_abs=current_profit_abs,
                     profit_fiat=current_profit_fiat,
 
+                    total_profit_abs=total_profit_abs,
+                    total_profit_fiat=total_profit_fiat,
                     stoploss_current_dist=stoploss_current_dist,
                     stoploss_current_dist_ratio=round(stoploss_current_dist_ratio, 8),
                     stoploss_current_dist_pct=round(stoploss_current_dist_ratio * 100, 2),
@@ -367,13 +373,13 @@ class RPC:
 
     def _rpc_trade_history(self, limit: int, offset: int = 0, order_by_id: bool = False) -> Dict:
         """ Returns the X last trades """
-        order_by = Trade.id if order_by_id else Trade.close_date.desc()
+        order_by: Any = Trade.id if order_by_id else Trade.close_date.desc()
         if limit:
             trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(
                 order_by).limit(limit).offset(offset)
         else:
             trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(
-                Trade.close_date.desc()).all()
+                Trade.close_date.desc())
 
         output = [trade.to_json() for trade in trades]
 
@@ -395,7 +401,7 @@ class RPC:
                 return 'losses'
             else:
                 return 'draws'
-        trades: List[Trade] = Trade.get_trades([Trade.is_open.is_(False)], include_orders=False)
+        trades = Trade.get_trades([Trade.is_open.is_(False)], include_orders=False)
         # Sell reason
         exit_reasons = {}
         for trade in trades:
@@ -404,7 +410,7 @@ class RPC:
             exit_reasons[trade.exit_reason][trade_win_loss(trade)] += 1
 
         # Duration
-        dur: Dict[str, List[int]] = {'wins': [], 'draws': [], 'losses': []}
+        dur: Dict[str, List[float]] = {'wins': [], 'draws': [], 'losses': []}
         for trade in trades:
             if trade.close_date is not None and trade.open_date is not None:
                 trade_dur = (trade.close_date - trade.open_date).total_seconds()
@@ -443,11 +449,11 @@ class RPC:
                 durations.append((trade.close_date - trade.open_date).total_seconds())
 
             if not trade.is_open:
-                profit_ratio = trade.close_profit
+                profit_ratio = trade.close_profit or 0.0
-                profit_abs = trade.close_profit_abs
+                profit_abs = trade.close_profit_abs or 0.0
                 profit_closed_coin.append(profit_abs)
                 profit_closed_ratio.append(profit_ratio)
-                if trade.close_profit >= 0:
+                if profit_ratio >= 0:
                     winning_trades += 1
                     winning_profit += profit_abs
                 else:
@@ -500,7 +506,7 @@ class RPC:
 
         trades_df = DataFrame([{'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT),
                                 'profit_abs': trade.close_profit_abs}
-                               for trade in trades if not trade.is_open])
+                               for trade in trades if not trade.is_open and trade.close_date])
         max_drawdown_abs = 0.0
         max_drawdown = 0.0
         if len(trades_df) > 0:
@@ -779,7 +785,8 @@ class RPC:
         # check if valid pair
 
         # check if pair already has an open pair
-        trade: Trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()
+        trade: Optional[Trade] = Trade.get_trades(
+            [Trade.is_open.is_(True), Trade.pair == pair]).first()
         is_short = (order_side == SignalDirection.SHORT)
         if trade:
             is_short = trade.is_short
@@ -1198,10 +1205,23 @@ class RPC:
             "ram_pct": psutil.virtual_memory().percent
         }
 
-    def _health(self) -> Dict[str, Union[str, int]]:
+    def health(self) -> Dict[str, Optional[Union[str, int]]]:
         last_p = self._freqtrade.last_process
+        if last_p is None:
+            return {
+                "last_process": None,
+                "last_process_loc": None,
+                "last_process_ts": None,
+            }
+
         return {
-            'last_process': str(last_p),
+            "last_process": str(last_p),
-            'last_process_loc': last_p.astimezone(tzlocal()).strftime(DATETIME_PRINT_FORMAT),
+            "last_process_loc": last_p.astimezone(tzlocal()).strftime(DATETIME_PRINT_FORMAT),
-            'last_process_ts': int(last_p.timestamp()),
+            "last_process_ts": int(last_p.timestamp()),
         }
+
+    def _update_market_direction(self, direction: MarketDirection) -> None:
+        self._freqtrade.strategy.market_direction = direction
+
+    def _get_market_direction(self) -> MarketDirection:
+        return self._freqtrade.strategy.market_direction

freqtrade/rpc/telegram.py

@@ -25,7 +25,7 @@ from telegram.utils.helpers import escape_markdown
 
 from freqtrade.__init__ import __version__
 from freqtrade.constants import DUST_PER_COIN, Config
-from freqtrade.enums import RPCMessageType, SignalDirection, TradingMode
+from freqtrade.enums import MarketDirection, RPCMessageType, SignalDirection, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import chunks, plural, round_coin_value
 from freqtrade.persistence import Trade
@@ -129,7 +129,8 @@ class Telegram(RPCHandler):
             r'/weekly$', r'/weekly \d+$', r'/monthly$', r'/monthly \d+$',
             r'/forcebuy$', r'/forcelong$', r'/forceshort$',
             r'/forcesell$', r'/forceexit$',
-            r'/edge$', r'/health$', r'/help$', r'/version$'
+            r'/edge$', r'/health$', r'/help$', r'/version$', r'/marketdir (long|short|even|none)$',
+            r'/marketdir$'
         ]
         # Create keys for generation
         valid_keys_print = [k.replace('$', '') for k in valid_keys]
@@ -197,6 +198,7 @@ class Telegram(RPCHandler):
             CommandHandler('health', self._health),
             CommandHandler('help', self._help),
             CommandHandler('version', self._version),
+            CommandHandler('marketdir', self._changemarketdir)
         ]
         callbacks = [
             CallbackQueryHandler(self._status_table, pattern='update_status_table'),
@@ -469,42 +471,47 @@ class Telegram(RPCHandler):
         lines_detail: List[str] = []
         if len(filled_orders) > 0:
             first_avg = filled_orders[0]["safe_price"]
-
+        order_nr = 0
-        for x, order in enumerate(filled_orders):
+        for order in filled_orders:
             lines: List[str] = []
             if order['is_open'] is True:
                 continue
+            order_nr += 1
             wording = 'Entry' if order['ft_is_entry'] else 'Exit'
 
             cur_entry_datetime = arrow.get(order["order_filled_date"])
             cur_entry_amount = order["filled"] or order["amount"]
             cur_entry_average = order["safe_price"]
             lines.append("  ")
-            if x == 0:
+            if order_nr == 1:
-                lines.append(f"*{wording} #{x+1}:*")
+                lines.append(f"*{wording} #{order_nr}:*")
                 lines.append(
                     f"*Amount:* {cur_entry_amount} ({order['cost']:.8f} {quote_currency})")
                 lines.append(f"*Average Price:* {cur_entry_average}")
             else:
-                sumA = 0
+                sum_stake = 0
-                sumB = 0
+                sum_amount = 0
-                for y in range(x):
+                for y in range(order_nr):
-                    amount = filled_orders[y]["filled"] or filled_orders[y]["amount"]
+                    loc_order = filled_orders[y]
-                    sumA += amount * filled_orders[y]["safe_price"]
+                    if loc_order['is_open'] is True:
-                    sumB += amount
+                        # Skip open orders (e.g. stop orders)
-                prev_avg_price = sumA / sumB
+                        continue
+                    amount = loc_order["filled"] or loc_order["amount"]
+                    sum_stake += amount * loc_order["safe_price"]
+                    sum_amount += amount
+                prev_avg_price = sum_stake / sum_amount
                 # TODO: This calculation ignores fees.
                 price_to_1st_entry = ((cur_entry_average - first_avg) / first_avg)
                 minus_on_entry = 0
                 if prev_avg_price:
                     minus_on_entry = (cur_entry_average - prev_avg_price) / prev_avg_price
 
-                lines.append(f"*{wording} #{x+1}:* at {minus_on_entry:.2%} avg profit")
+                lines.append(f"*{wording} #{order_nr}:* at {minus_on_entry:.2%} avg profit")
                 if is_open:
                     lines.append("({})".format(cur_entry_datetime
                                                .humanize(granularity=["day", "hour", "minute"])))
-                lines.append(
+                lines.append(f"*Amount:* {cur_entry_amount} "
-                    f"*Amount:* {cur_entry_amount} ({order['cost']:.8f} {quote_currency})")
+                             f"({round_coin_value(order['cost'], quote_currency)})")
                 lines.append(f"*Average {wording} Price:* {cur_entry_average} "
                              f"({price_to_1st_entry:.2%} from 1st entry rate)")
                 lines.append(f"*Order filled:* {order['order_filled_date']}")
@@ -518,6 +525,7 @@ class Telegram(RPCHandler):
                 # lines.append(
                 # f"({days}d {hours}h {minutes}m {seconds}s from previous {wording.lower()})")
             lines_detail.append("\n".join(lines))
+
         return lines_detail
 
     @authorized_only
@@ -553,14 +561,21 @@ class Telegram(RPCHandler):
         for r in results:
             r['open_date_hum'] = arrow.get(r['open_date']).humanize()
             r['num_entries'] = len([o for o in r['orders'] if o['ft_is_entry']])
+            r['num_exits'] = len([o for o in r['orders'] if not o['ft_is_entry']
+                                 and not o['ft_order_side'] == 'stoploss'])
             r['exit_reason'] = r.get('exit_reason', "")
+            r['stake_amount_r'] = round_coin_value(r['stake_amount'], r['quote_currency'])
+            r['profit_abs_r'] = round_coin_value(r['profit_abs'], r['quote_currency'])
+            r['realized_profit_r'] = round_coin_value(r['realized_profit'], r['quote_currency'])
+            r['total_profit_abs_r'] = round_coin_value(
+                r['total_profit_abs'], r['quote_currency'])
             lines = [
                 "*Trade ID:* `{trade_id}`" +
                 (" `(since {open_date_hum})`" if r['is_open'] else ""),
                 "*Current Pair:* {pair}",
-                "*Direction:* " + ("`Short`" if r.get('is_short') else "`Long`"),
+                f"*Direction:* {'`Short`' if r.get('is_short') else '`Long`'}"
-                "*Leverage:* `{leverage}`" if r.get('leverage') else "",
+                + " ` ({leverage}x)`" if r.get('leverage') else "",
-                "*Amount:* `{amount} ({stake_amount} {quote_currency})`",
+                "*Amount:* `{amount} ({stake_amount_r})`",
                 "*Enter Tag:* `{enter_tag}`" if r['enter_tag'] else "",
                 "*Exit Reason:* `{exit_reason}`" if r['exit_reason'] else "",
             ]
@@ -568,6 +583,7 @@ class Telegram(RPCHandler):
             if position_adjust:
                 max_buy_str = (f"/{max_entries + 1}" if (max_entries > 0) else "")
                 lines.append("*Number of Entries:* `{num_entries}`" + max_buy_str)
+                lines.append("*Number of Exits:* `{num_exits}`")
 
             lines.extend([
                 "*Open Rate:* `{open_rate:.8f}`",
@@ -575,13 +591,15 @@ class Telegram(RPCHandler):
                 "*Open Date:* `{open_date}`",
                 "*Close Date:* `{close_date}`" if r['close_date'] else "",
                 "*Current Rate:* `{current_rate:.8f}`" if r['is_open'] else "",
-                ("*Current Profit:* " if r['is_open'] else "*Close Profit: *")
+                ("*Unrealized Profit:* " if r['is_open'] else "*Close Profit: *")
-                + "`{profit_ratio:.2%}`",
+                + "`{profit_ratio:.2%}` `({profit_abs_r})`",
             ])
 
             if r['is_open']:
                 if r.get('realized_profit'):
-                    lines.append("*Realized Profit:* `{realized_profit:.8f}`")
+                    lines.append("*Realized Profit:* `{realized_profit_r}`")
+                    lines.append("*Total Profit:* `{total_profit_abs_r}` ")
+
                 if (r['stop_loss_abs'] != r['initial_stop_loss_abs']
                         and r['initial_stop_loss_ratio'] is not None):
                     # Adding initial stoploss only if it is different from stoploss
@@ -1040,10 +1058,14 @@ class Telegram(RPCHandler):
                     query.answer()
                     query.edit_message_text(text="Force exit canceled.")
                     return
-                trade: Trade = Trade.get_trades(trade_filter=Trade.id == trade_id).first()
+                trade: Optional[Trade] = Trade.get_trades(trade_filter=Trade.id == trade_id).first()
                 query.answer()
-                query.edit_message_text(text=f"Manually exiting Trade #{trade_id}, {trade.pair}")
+                if trade:
-                self._force_exit_action(trade_id)
+                    query.edit_message_text(
+                        text=f"Manually exiting Trade #{trade_id}, {trade.pair}")
+                    self._force_exit_action(trade_id)
+                else:
+                    query.edit_message_text(text=f"Trade {trade_id} not found.")
 
     def _force_enter_action(self, pair, price: Optional[float], order_side: SignalDirection):
         if pair != 'cancel':
@@ -1494,6 +1516,9 @@ class Telegram(RPCHandler):
             "*/count:* `Show number of active trades compared to allowed number of trades`\n"
             "*/edge:* `Shows validated pairs by Edge if it is enabled` \n"
             "*/health* `Show latest process timestamp - defaults to 1970-01-01 00:00:00` \n"
+            "*/marketdir [long | short | even | none]:* `Updates the user managed variable "
+            "that represents the current market direction. If no direction is provided `"
+            "`the currently set market direction will be output.` \n"
 
             "_Statistics_\n"
             "------------\n"
@@ -1527,7 +1552,7 @@ class Telegram(RPCHandler):
         Handler for /health
         Shows the last process timestamp
         """
-        health = self._rpc._health()
+        health = self._rpc.health()
         message = f"Last process: `{health['last_process_loc']}`"
         self._send_msg(message)
 
@@ -1677,3 +1702,39 @@ class Telegram(RPCHandler):
                 'TelegramError: %s! Giving up on that message.',
                 telegram_err.message
             )
+
+    @authorized_only
+    def _changemarketdir(self, update: Update, context: CallbackContext) -> None:
+        """
+        Handler for /marketdir.
+        Updates the bot's market_direction
+        :param bot: telegram bot
+        :param update: message update
+        :return: None
+        """
+        if context.args and len(context.args) == 1:
+            new_market_dir_arg = context.args[0]
+            old_market_dir = self._rpc._get_market_direction()
+            new_market_dir = None
+            if new_market_dir_arg == "long":
+                new_market_dir = MarketDirection.LONG
+            elif new_market_dir_arg == "short":
+                new_market_dir = MarketDirection.SHORT
+            elif new_market_dir_arg == "even":
+                new_market_dir = MarketDirection.EVEN
+            elif new_market_dir_arg == "none":
+                new_market_dir = MarketDirection.NONE
+
+            if new_market_dir is not None:
+                self._rpc._update_market_direction(new_market_dir)
+                self._send_msg("Successfully updated market direction"
+                               f" from *{old_market_dir}* to *{new_market_dir}*.")
+            else:
+                raise RPCException("Invalid market direction provided. \n"
+                                   "Valid market directions: *long, short, even, none*")
+        elif context.args is not None and len(context.args) == 0:
+            old_market_dir = self._rpc._get_market_direction()
+            self._send_msg(f"Currently set market direction: *{old_market_dir}*")
+        else:
+            raise RPCException("Invalid usage of command /marketdir. \n"
+                               "Usage: */marketdir [short |  long | even | none]*")

freqtrade/strategy/interface.py

@@ -12,8 +12,8 @@ from pandas import DataFrame
 
 from freqtrade.constants import Config, IntOrInf, ListPairsWithTimeframes
 from freqtrade.data.dataprovider import DataProvider
-from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, RunMode, SignalDirection,
+from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, MarketDirection, RunMode,
-                             SignalTagType, SignalType, TradingMode)
+                             SignalDirection, SignalTagType, SignalType, TradingMode)
 from freqtrade.exceptions import OperationalException, StrategyError
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date, timeframe_to_seconds
 from freqtrade.misc import remove_entry_exit_signals
@@ -122,6 +122,9 @@ class IStrategy(ABC, HyperStrategyMixin):
     # Definition of plot_config. See plotting documentation for more details.
     plot_config: Dict = {}
 
+    # A self set parameter that represents the market direction. filled from configuration
+    market_direction: MarketDirection = MarketDirection.NONE
+
     def __init__(self, config: Config) -> None:
         self.config = config
         # Dict to determine if analysis is necessary

freqtrade/templates/FreqaiExampleHybridStrategy.py

@@ -1,13 +1,13 @@
 import logging
 from typing import Dict
 
-import numpy as np
+import numpy as np  # noqa
-import pandas as pd
+import pandas as pd  # noqa
 import talib.abstract as ta
 from pandas import DataFrame
 from technical import qtpylib
 
-from freqtrade.strategy import IntParameter, IStrategy, merge_informative_pair
+from freqtrade.strategy import IntParameter, IStrategy, merge_informative_pair  # noqa
 
 
 logger = logging.getLogger(__name__)
@@ -27,7 +27,7 @@ class FreqaiExampleHybridStrategy(IStrategy):
 
     "freqai": {
         "enabled": true,
-        "purge_old_models": true,
+        "purge_old_models": 2,
         "train_period_days": 15,
         "identifier": "uniqe-id",
         "feature_parameters": {
@@ -224,12 +224,11 @@ class FreqaiExampleHybridStrategy(IStrategy):
         usage example: dataframe["&-target"] = dataframe["close"].shift(-1) / dataframe["close"]
         """
         dataframe['&s-up_or_down'] = np.where(dataframe["close"].shift(-50) >
-                                        dataframe["close"], 'up', 'down')
+                                              dataframe["close"], 'up', 'down')
 
         return dataframe
 
-    # flake8: noqa: C901
+    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:  # noqa: C901
-    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
 
         # User creates their own custom strat here. Present example is a supertrend
         # based strategy.

freqtrade/util/__init__.py

@@ -1,3 +1,2 @@
-# flake8: noqa: F401
+from freqtrade.util.ft_precise import FtPrecise  # noqa: F401
-from freqtrade.util.ft_precise import FtPrecise
+from freqtrade.util.periodic_cache import PeriodicCache  # noqa: F401
-from freqtrade.util.periodic_cache import PeriodicCache

freqtrade/vendor/qtpylib/indicators.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # QTPyLib: Quantitative Trading Python Library

pyproject.toml

@@ -35,6 +35,9 @@ warn_unused_ignores = true
 exclude = [
     '^build_helpers\.py$'
 ]
+plugins = [
+  "sqlalchemy.ext.mypy.plugin"
+]
 
 [[tool.mypy.overrides]]
 module = "tests.*"
@@ -56,3 +59,24 @@ exclude = [
   "build_helpers/*.py",
 ]
 ignore = ["freqtrade/vendor/**"]
+
+
+[tool.ruff]
+line-length = 100
+extend-exclude = [".env"]
+target-version = "py38"
+extend-select = [
+  "C90",    # mccabe
+  # "N",    # pep8-naming
+  # "UP",   # pyupgrade
+  "TID",    # flake8-tidy-imports
+  # "EXE",  # flake8-executable
+  "YTT",    # flake8-2020
+  # "DTZ",  # flake8-datetimez
+  # "RSE",  # flake8-raise
+  # "TCH",  # flake8-type-checking
+  "PTH",  # flake8-use-pathlib
+]
+
+[tool.ruff.mccabe]
+max-complexity = 12

requirements-dev.txt

@@ -7,10 +7,9 @@
 -r docs/requirements-docs.txt
 
 coveralls==3.3.1
-flake8==6.0.0
+ruff==0.0.253
-flake8-tidy-imports==4.8.0
+mypy==1.0.1
-mypy==1.0.0
+pre-commit==3.1.1
-pre-commit==3.0.4
 pytest==7.2.1
 pytest-asyncio==0.20.3
 pytest-cov==4.0.0
@@ -26,8 +25,8 @@ httpx==0.23.3
 nbconvert==7.2.9
 
 # mypy types
-types-cachetools==5.3.0.0
+types-cachetools==5.3.0.4
 types-filelock==3.2.7
-types-requests==2.28.11.12
+types-requests==2.28.11.15
-types-tabulate==0.9.0.0
+types-tabulate==0.9.0.1
-types-python-dateutil==2.8.19.6
+types-python-dateutil==2.8.19.9

requirements-freqai-rl.txt

@@ -2,9 +2,9 @@
 -r requirements-freqai.txt
 
 # Required for freqai-rl
-torch==1.13.1
+torch==1.13.1; python_version < '3.11'
-stable-baselines3==1.7.0
+stable-baselines3==1.7.0; python_version < '3.11'
-sb3-contrib==1.7.0
+sb3-contrib==1.7.0; python_version < '3.11'
 # Gym is forced to this version by stable-baselines3.
 setuptools==65.5.1 # Should be removed when gym is fixed.
-gym==0.21
+gym==0.21; python_version < '3.11'

requirements-freqai.txt

@@ -5,7 +5,7 @@
 # Required for freqai
 scikit-learn==1.1.3
 joblib==1.2.0
-catboost==1.1.1; platform_machine != 'aarch64'
+catboost==1.1.1; platform_machine != 'aarch64' and python_version < '3.11'
 lightgbm==3.3.5
-xgboost==1.7.3
+xgboost==1.7.4
 tensorboard==2.12.0

requirements-hyperopt.txt

@@ -2,7 +2,7 @@
 -r requirements.txt
 
 # Required for hyperopt
-scipy==1.10.0
+scipy==1.10.1
 scikit-learn==1.1.3
 scikit-optimize==0.9.0
 filelock==3.9.0

requirements-plot.txt

@@ -1,4 +1,4 @@
 # Include all requirements to run the bot.
 -r requirements.txt
 
-plotly==5.13.0
+plotly==5.13.1

requirements.txt

@@ -2,12 +2,10 @@ numpy==1.24.2
 pandas==1.5.3
 pandas-ta==0.3.14b
 
-ccxt==2.7.93
+ccxt==2.8.54
-# Pin cryptography for now due to rust build errors with piwheels
+cryptography==39.0.1
-cryptography==38.0.1; platform_machine == 'armv7l'
-cryptography==39.0.1; platform_machine != 'armv7l'
 aiohttp==3.8.4
-SQLAlchemy==1.4.46
+SQLAlchemy==2.0.4
 python-telegram-bot==13.15
 arrow==1.2.3
 cachetools==4.2.2
@@ -36,8 +34,8 @@ orjson==3.8.6
 sdnotify==0.3.2
 
 # API Server
-fastapi==0.91.0
+fastapi==0.92.0
-pydantic==1.10.4
+pydantic==1.10.5
 uvicorn==0.20.0
 pyjwt==2.6.0
 aiofiles==23.1.0
@@ -47,7 +45,7 @@ psutil==5.9.4
 colorama==0.4.6
 # Building config files interactively
 questionary==1.10.0
-prompt-toolkit==3.0.36
+prompt-toolkit==3.0.37
 # Extensions to datetime library
 python-dateutil==2.8.2
 

setup.cfg

@@ -17,6 +17,7 @@ classifiers =
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
     Operating System :: MacOS
     Operating System :: Unix
     Topic :: Office/Business :: Financial :: Investment

setup.py

@@ -32,8 +32,6 @@ hdf5 = [
 
 develop = [
     'coveralls',
-    'flake8',
-    'flake8-tidy-imports',
     'mypy',
     'pytest',
     'pytest-asyncio',

tests/commands/test_commands.py

@@ -24,7 +24,7 @@ from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.persistence.models import init_db
 from freqtrade.persistence.pairlock_middleware import PairLocks
-from tests.conftest import (CURRENT_TEST_STRATEGY, create_mock_trades, get_args, log_has,
+from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, create_mock_trades, get_args, log_has,
                             log_has_re, patch_exchange, patched_configuration_load_config_file)
 from tests.conftest_trades import MOCK_TRADE_COUNT
 
@@ -454,7 +454,7 @@ def test_list_markets(mocker, markets_static, capsys):
     assert re.search(r"^BLK/BTC$", captured.out, re.MULTILINE)
     assert re.search(r"^LTC/USD$", captured.out, re.MULTILINE)
 
-    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(side_effect=ValueError))
+    mocker.patch(f'{EXMS}.markets', PropertyMock(side_effect=ValueError))
     # Test --one-column
     args = [
         "list-markets",
@@ -643,9 +643,7 @@ def test_download_data_keyboardInterrupt(mocker, markets):
     dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data',
                            MagicMock(side_effect=KeyboardInterrupt))
     patch_exchange(mocker)
-    mocker.patch(
+    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value=markets))
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
-    )
     args = [
         "download-data",
         "--exchange", "binance",
@@ -664,9 +662,7 @@ def test_download_data_timerange(mocker, markets):
     dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data',
                            MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
     patch_exchange(mocker)
-    mocker.patch(
+    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value=markets))
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
-    )
     args = [
         "download-data",
         "--exchange", "binance",
@@ -715,9 +711,7 @@ def test_download_data_no_markets(mocker, caplog):
     dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data',
                            MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
     patch_exchange(mocker, id='binance')
-    mocker.patch(
+    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value={}))
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
-    )
     args = [
         "download-data",
         "--exchange", "binance",
@@ -733,9 +727,7 @@ def test_download_data_no_exchange(mocker, caplog):
     mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data',
                  MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
     patch_exchange(mocker)
-    mocker.patch(
+    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value={}))
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
-    )
     args = [
         "download-data",
     ]
@@ -751,9 +743,7 @@ def test_download_data_no_pairs(mocker):
     mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data',
                  MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
     patch_exchange(mocker)
-    mocker.patch(
+    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value={}))
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
-    )
     args = [
         "download-data",
         "--exchange",
@@ -771,9 +761,7 @@ def test_download_data_all_pairs(mocker, markets):
     dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data',
                            MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
     patch_exchange(mocker)
-    mocker.patch(
+    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value=markets))
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
-    )
     args = [
         "download-data",
         "--exchange",
@@ -810,9 +798,7 @@ def test_download_data_trades(mocker, caplog):
     convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv',
                                 MagicMock(return_value=[]))
     patch_exchange(mocker)
-    mocker.patch(
+    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value={}))
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
-    )
     args = [
         "download-data",
         "--exchange", "kraken",
@@ -843,9 +829,7 @@ def test_download_data_trades(mocker, caplog):
 
 def test_download_data_data_invalid(mocker):
     patch_exchange(mocker, id="kraken")
-    mocker.patch(
+    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value={}))
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
-    )
     args = [
         "download-data",
         "--exchange", "kraken",
@@ -862,9 +846,7 @@ def test_start_convert_trades(mocker, caplog):
     convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv',
                                 MagicMock(return_value=[]))
     patch_exchange(mocker)
-    mocker.patch(
+    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value={}))
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
-    )
     args = [
         "trades-to-ohlcv",
         "--exchange", "kraken",
@@ -971,7 +953,7 @@ def test_start_list_freqAI_models(capsys):
 
 def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys):
     patch_exchange(mocker, mock_markets=True)
-    mocker.patch.multiple('freqtrade.exchange.Exchange',
+    mocker.patch.multiple(EXMS,
                           exchange_has=MagicMock(return_value=True),
                           get_tickers=tickers,
                           )

tests/conftest.py

@@ -40,6 +40,7 @@ np.seterr(all='raise')
 
 CURRENT_TEST_STRATEGY = 'StrategyTestV3'
 TRADE_SIDES = ('long', 'short')
+EXMS = 'freqtrade.exchange.exchange.Exchange'
 
 
 def pytest_addoption(parser):
@@ -145,22 +146,21 @@ def patch_exchange(
     mock_markets=True,
     mock_supported_modes=True
 ) -> None:
-    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock(return_value={}))
+    mocker.patch(f'{EXMS}._load_async_markets', return_value={})
-    mocker.patch('freqtrade.exchange.Exchange.validate_config', MagicMock())
+    mocker.patch(f'{EXMS}.validate_config', MagicMock())
-    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
+    mocker.patch(f'{EXMS}.validate_timeframes', MagicMock())
-    mocker.patch('freqtrade.exchange.Exchange.id', PropertyMock(return_value=id))
+    mocker.patch(f'{EXMS}.id', PropertyMock(return_value=id))
-    mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value=id.title()))
+    mocker.patch(f'{EXMS}.name', PropertyMock(return_value=id.title()))
-    mocker.patch('freqtrade.exchange.Exchange.precisionMode', PropertyMock(return_value=2))
+    mocker.patch(f'{EXMS}.precisionMode', PropertyMock(return_value=2))
 
     if mock_markets:
         if isinstance(mock_markets, bool):
             mock_markets = get_markets()
-        mocker.patch('freqtrade.exchange.Exchange.markets',
+        mocker.patch(f'{EXMS}.markets', PropertyMock(return_value=mock_markets))
-                     PropertyMock(return_value=mock_markets))
 
     if mock_supported_modes:
         mocker.patch(
-            f'freqtrade.exchange.{id.capitalize()}._supported_trading_mode_margin_pairs',
+            f'freqtrade.exchange.{id}.{id.capitalize()}._supported_trading_mode_margin_pairs',
             PropertyMock(return_value=[
                 (TradingMode.MARGIN, MarginMode.CROSS),
                 (TradingMode.MARGIN, MarginMode.ISOLATED),
@@ -170,10 +170,10 @@ def patch_exchange(
         )
 
     if api_mock:
-        mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
+        mocker.patch(f'{EXMS}._init_ccxt', return_value=api_mock)
     else:
-        mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock())
+        mocker.patch(f'{EXMS}._init_ccxt', MagicMock())
-        mocker.patch('freqtrade.exchange.Exchange.timeframes', PropertyMock(
+        mocker.patch(f'{EXMS}.timeframes', PropertyMock(
                 return_value=['5m', '15m', '1h', '1d']))
 
 
@@ -2573,7 +2573,7 @@ def import_fails() -> None:
     realimport = builtins.__import__
 
     def mockedimport(name, *args, **kwargs):
-        if name in ["filelock", 'systemd.journal', 'uvloop']:
+        if name in ["filelock", 'cysystemd.journal', 'uvloop']:
             raise ImportError(f"No module named '{name}'")
         return realimport(name, *args, **kwargs)
 

tests/data/test_dataprovider.py

@@ -8,7 +8,7 @@ from freqtrade.data.dataprovider import DataProvider
 from freqtrade.enums import CandleType, RunMode
 from freqtrade.exceptions import ExchangeError, OperationalException
 from freqtrade.plugins.pairlistmanager import PairListManager
-from tests.conftest import generate_test_data, get_patched_exchange
+from tests.conftest import EXMS, generate_test_data, get_patched_exchange
 
 
 @pytest.mark.parametrize('candle_type', [
@@ -223,7 +223,7 @@ def test_emit_df(mocker, default_conf, ohlcv_history):
 
 def test_refresh(mocker, default_conf):
     refresh_mock = MagicMock()
-    mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock)
+    mocker.patch(f"{EXMS}.refresh_latest_ohlcv", refresh_mock)
 
     exchange = get_patched_exchange(mocker, default_conf, id="binance")
     timeframe = default_conf["timeframe"]
@@ -281,7 +281,7 @@ def test_market(mocker, default_conf, markets):
 
 def test_ticker(mocker, default_conf, tickers):
     ticker_mock = MagicMock(return_value=tickers()['ETH/BTC'])
-    mocker.patch("freqtrade.exchange.Exchange.fetch_ticker", ticker_mock)
+    mocker.patch(f"{EXMS}.fetch_ticker", ticker_mock)
     exchange = get_patched_exchange(mocker, default_conf)
     dp = DataProvider(default_conf, exchange)
     res = dp.ticker('ETH/BTC')
@@ -290,7 +290,7 @@ def test_ticker(mocker, default_conf, tickers):
     assert res['symbol'] == 'ETH/BTC'
 
     ticker_mock = MagicMock(side_effect=ExchangeError('Pair not found'))
-    mocker.patch("freqtrade.exchange.Exchange.fetch_ticker", ticker_mock)
+    mocker.patch(f"{EXMS}.fetch_ticker", ticker_mock)
     exchange = get_patched_exchange(mocker, default_conf)
     dp = DataProvider(default_conf, exchange)
     res = dp.ticker('UNITTEST/BTC')
@@ -301,7 +301,7 @@ def test_current_whitelist(mocker, default_conf, tickers):
     # patch default conf to volumepairlist
     default_conf['pairlists'][0] = {'method': 'VolumePairList', "number_assets": 5}
 
-    mocker.patch.multiple('freqtrade.exchange.Exchange',
+    mocker.patch.multiple(EXMS,
                           exchange_has=MagicMock(return_value=True),
                           get_tickers=tickers)
     exchange = get_patched_exchange(mocker, default_conf)

tests/data/test_history.py

@@ -26,7 +26,7 @@ from freqtrade.enums import CandleType
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import file_dump_json
 from freqtrade.resolvers import StrategyResolver
-from tests.conftest import (CURRENT_TEST_STRATEGY, get_patched_exchange, log_has, log_has_re,
+from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, get_patched_exchange, log_has, log_has_re,
                             patch_exchange)
 
 
@@ -66,7 +66,7 @@ def test_load_data_7min_timeframe(caplog, testdatadir) -> None:
 
 
 def test_load_data_1min_timeframe(ohlcv_history, mocker, caplog, testdatadir) -> None:
-    mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history)
+    mocker.patch(f'{EXMS}.get_historic_ohlcv', return_value=ohlcv_history)
     file = testdatadir / 'UNITTEST_BTC-1m.json'
     load_data(datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC'])
     assert file.is_file()
@@ -77,7 +77,7 @@ def test_load_data_1min_timeframe(ohlcv_history, mocker, caplog, testdatadir) ->
 
 
 def test_load_data_mark(ohlcv_history, mocker, caplog, testdatadir) -> None:
-    mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history)
+    mocker.patch(f'{EXMS}.get_historic_ohlcv', return_value=ohlcv_history)
     file = testdatadir / 'futures/UNITTEST_USDT_USDT-1h-mark.json'
     load_data(datadir=testdatadir, timeframe='1h', pairs=['UNITTEST/BTC'], candle_type='mark')
     assert file.is_file()
@@ -109,7 +109,7 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog,
     Test load_pair_history() with 1 min timeframe
     """
     tmpdir1 = Path(tmpdir)
-    mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history_list)
+    mocker.patch(f'{EXMS}.get_historic_ohlcv', return_value=ohlcv_history_list)
     exchange = get_patched_exchange(mocker, default_conf)
     file = tmpdir1 / 'MEME_BTC-1m.json'
 
@@ -191,7 +191,7 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None:
 
     test_data = None
     test_filename = testdatadir.joinpath('UNITTEST_BTC-1m.json')
-    with open(test_filename, "rt") as file:
+    with test_filename.open("rt") as file:
         test_data = json.load(file)
 
     test_data_df = ohlcv_to_dataframe(test_data, '1m', 'UNITTEST/BTC',
@@ -277,7 +277,7 @@ def test_download_pair_history(
     subdir,
     file_tail
 ) -> None:
-    mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history_list)
+    mocker.patch(f'{EXMS}.get_historic_ohlcv', return_value=ohlcv_history_list)
     exchange = get_patched_exchange(mocker, default_conf)
     tmpdir1 = Path(tmpdir)
     file1_1 = tmpdir1 / f'{subdir}MEME_BTC-1m{file_tail}.json'
@@ -328,7 +328,7 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None:
     json_dump_mock = mocker.patch(
         'freqtrade.data.history.jsondatahandler.JsonDataHandler.ohlcv_store',
         return_value=None)
-    mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick)
+    mocker.patch(f'{EXMS}.get_historic_ohlcv', return_value=tick)
     exchange = get_patched_exchange(mocker, default_conf)
     _download_pair_history(datadir=testdatadir, exchange=exchange, pair="UNITTEST/BTC",
                            timeframe='1m', candle_type='spot')
@@ -340,7 +340,7 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None:
 
 
 def test_download_backtesting_data_exception(mocker, caplog, default_conf, tmpdir) -> None:
-    mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv',
+    mocker.patch(f'{EXMS}.get_historic_ohlcv',
                  side_effect=Exception('File Error'))
     tmpdir1 = Path(tmpdir)
     exchange = get_patched_exchange(mocker, default_conf)
@@ -506,9 +506,7 @@ def test_refresh_backtest_ohlcv_data(
         mocker, default_conf, markets, caplog, testdatadir, trademode, callcount):
     dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history',
                            MagicMock())
-    mocker.patch(
+    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value=markets))
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
-    )
     mocker.patch.object(Path, "exists", MagicMock(return_value=True))
     mocker.patch.object(Path, "unlink", MagicMock())
 
@@ -531,9 +529,7 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
                            MagicMock())
 
     ex = get_patched_exchange(mocker, default_conf)
-    mocker.patch(
+    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value={}))
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
-    )
     timerange = TimeRange.parse_timerange("20190101-20190102")
     unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["BTT/BTC", "LTC/USDT"],
                                              timeframes=["1m", "5m"],
@@ -551,9 +547,7 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
 def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir):
     dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_trades_history',
                            MagicMock())
-    mocker.patch(
+    mocker.patch(f'{EXMS}.markets', PropertyMock(return_value=markets))
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
-    )
     mocker.patch.object(Path, "exists", MagicMock(return_value=True))
     mocker.patch.object(Path, "unlink", MagicMock())
 
@@ -577,8 +571,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
                                  tmpdir) -> None:
     tmpdir1 = Path(tmpdir)
     ght_mock = MagicMock(side_effect=lambda pair, *args, **kwargs: (pair, trades_history))
-    mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
+    mocker.patch(f'{EXMS}.get_historic_trades', ght_mock)
-                 ght_mock)
     exchange = get_patched_exchange(mocker, default_conf)
     file1 = tmpdir1 / 'ETH_BTC-trades.json.gz'
     data_handler = get_datahandler(tmpdir1, data_format='jsongz')
@@ -604,8 +597,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
 
     file1.unlink()
 
-    mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
+    mocker.patch(f'{EXMS}.get_historic_trades', MagicMock(side_effect=ValueError))
-                 MagicMock(side_effect=ValueError))
 
     assert not _download_trades_history(data_handler=data_handler, exchange=exchange,
                                         pair='ETH/BTC')
@@ -615,8 +607,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
     copyfile(testdatadir / file2.name, file2)
 
     ght_mock.reset_mock()
-    mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
+    mocker.patch(f'{EXMS}.get_historic_trades', ght_mock)
-                 ght_mock)
     # Since before first start date
     since_time = int(trades_history[0][0] // 1000) - 500
     timerange = TimeRange('date', None, since_time, 0)

tests/edge/test_edge.py

@@ -14,7 +14,7 @@ from freqtrade.data.converter import ohlcv_to_dataframe
 from freqtrade.edge import Edge, PairInfo
 from freqtrade.enums import ExitType
 from freqtrade.exceptions import OperationalException
-from tests.conftest import get_patched_freqtradebot, log_has
+from tests.conftest import EXMS, get_patched_freqtradebot, log_has
 from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe,
                             _get_frame_time_from_offset)
 
@@ -261,7 +261,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m',
 
 def test_edge_process_downloaded_data(mocker, edge_conf):
     freqtrade = get_patched_freqtradebot(mocker, edge_conf)
-    mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001))
+    mocker.patch(f'{EXMS}.get_fee', MagicMock(return_value=0.001))
     mocker.patch('freqtrade.edge.edge_positioning.refresh_data', MagicMock())
     mocker.patch('freqtrade.edge.edge_positioning.load_data', mocked_load_data)
     edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy)
@@ -273,7 +273,7 @@ def test_edge_process_downloaded_data(mocker, edge_conf):
 
 def test_edge_process_no_data(mocker, edge_conf, caplog):
     freqtrade = get_patched_freqtradebot(mocker, edge_conf)
-    mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001))
+    mocker.patch(f'{EXMS}.get_fee', MagicMock(return_value=0.001))
     mocker.patch('freqtrade.edge.edge_positioning.refresh_data', MagicMock())
     mocker.patch('freqtrade.edge.edge_positioning.load_data', MagicMock(return_value={}))
     edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy)
@@ -286,7 +286,7 @@ def test_edge_process_no_data(mocker, edge_conf, caplog):
 
 def test_edge_process_no_trades(mocker, edge_conf, caplog):
     freqtrade = get_patched_freqtradebot(mocker, edge_conf)
-    mocker.patch('freqtrade.exchange.Exchange.get_fee', return_value=0.001)
+    mocker.patch(f'{EXMS}.get_fee', return_value=0.001)
     mocker.patch('freqtrade.edge.edge_positioning.refresh_data', )
     mocker.patch('freqtrade.edge.edge_positioning.load_data', mocked_load_data)
     # Return empty
@@ -303,7 +303,7 @@ def test_edge_process_no_pairs(mocker, edge_conf, caplog):
     mocker.patch('freqtrade.freqtradebot.validate_config_consistency')
 
     freqtrade = get_patched_freqtradebot(mocker, edge_conf)
-    fee_mock = mocker.patch('freqtrade.exchange.Exchange.get_fee', return_value=0.001)
+    fee_mock = mocker.patch(f'{EXMS}.get_fee', return_value=0.001)
     mocker.patch('freqtrade.edge.edge_positioning.refresh_data')
     mocker.patch('freqtrade.edge.edge_positioning.load_data', mocked_load_data)
     # Return empty
@@ -319,7 +319,7 @@ def test_edge_process_no_pairs(mocker, edge_conf, caplog):
 
 def test_edge_init_error(mocker, edge_conf,):
     edge_conf['stake_amount'] = 0.5
-    mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001))
+    mocker.patch(f'{EXMS}.get_fee', MagicMock(return_value=0.001))
     with pytest.raises(OperationalException,  match='Edge works only with unlimited stake amount'):
         get_patched_freqtradebot(mocker, edge_conf)
 

tests/exchange/test_binance.py

@@ -7,7 +7,7 @@ import pytest
 
 from freqtrade.enums import CandleType, MarginMode, TradingMode
 from freqtrade.exceptions import DependencyException, InvalidOrderException, OperationalException
-from tests.conftest import get_mock_coro, get_patched_exchange, log_has_re
+from tests.conftest import EXMS, get_mock_coro, get_patched_exchange, log_has_re
 from tests.exchange.test_exchange import ccxt_exceptionhandlers
 
 
@@ -34,8 +34,8 @@ def test_create_stoploss_order_binance(default_conf, mocker, limitratio, expecte
     default_conf['dry_run'] = False
     default_conf['margin_mode'] = MarginMode.ISOLATED
     default_conf['trading_mode'] = trademode
-    mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y: y)
 
     exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance')
 
@@ -113,8 +113,8 @@ def test_create_stoploss_order_dry_run_binance(default_conf, mocker):
     api_mock = MagicMock()
     order_type = 'stop_loss_limit'
     default_conf['dry_run'] = True
-    mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y: y)
 
     exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance')
 
@@ -600,7 +600,7 @@ def test_get_maintenance_ratio_and_amt_binance(
     mm_ratio,
     amt,
 ):
-    mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True)
+    mocker.patch(f'{EXMS}.exchange_has', return_value=True)
     exchange = get_patched_exchange(mocker, default_conf, id="binance")
     exchange._leverage_tiers = leverage_tiers
     (result_ratio, result_amt) = exchange.get_maintenance_ratio_and_amt(pair, nominal_value)

tests/exchange/test_bitpanda.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from unittest.mock import MagicMock
 
-from tests.conftest import get_patched_exchange
+from tests.conftest import EXMS, get_patched_exchange
 
 
 def test_get_trades_for_order(default_conf, mocker):
@@ -9,7 +9,7 @@ def test_get_trades_for_order(default_conf, mocker):
     order_id = 'ABCD-ABCD'
     since = datetime(2018, 5, 5, 0, 0, 0)
     default_conf["dry_run"] = False
-    mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True)
+    mocker.patch(f'{EXMS}.exchange_has', return_value=True)
     api_mock = MagicMock()
 
     api_mock.fetch_my_trades = MagicMock(return_value=[{'id': 'TTR67E-3PFBD-76IISV',

tests/exchange/test_bybit.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timezone
 from unittest.mock import MagicMock
 
 from freqtrade.enums.marginmode import MarginMode
@@ -55,3 +56,19 @@ async def test_bybit_fetch_funding_rate(default_conf, mocker):
     kwargs = api_mock.fetch_funding_rate_history.call_args_list[0][1]
     assert kwargs['params'] == {'until': since_ms_end}
     assert kwargs['since'] == since_ms
+
+
+def test_bybit_get_funding_fees(default_conf, mocker):
+    now = datetime.now(timezone.utc)
+    exchange = get_patched_exchange(mocker, default_conf, id='bybit')
+    exchange._fetch_and_calculate_funding_fees = MagicMock()
+    exchange.get_funding_fees('BTC/USDT:USDT', 1, False, now)
+    assert exchange._fetch_and_calculate_funding_fees.call_count == 0
+
+    default_conf['trading_mode'] = 'futures'
+    default_conf['margin_mode'] = 'isolated'
+    exchange = get_patched_exchange(mocker, default_conf, id='bybit')
+    exchange._fetch_and_calculate_funding_fees = MagicMock()
+    exchange.get_funding_fees('BTC/USDT:USDT', 1, False, now)
+
+    assert exchange._fetch_and_calculate_funding_fees.call_count == 1

tests/exchange/test_ccxt_compat.py

@@ -17,7 +17,7 @@ from freqtrade.enums import CandleType
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
 from freqtrade.exchange.exchange import Exchange, timeframe_to_msecs
 from freqtrade.resolvers.exchange_resolver import ExchangeResolver
-from tests.conftest import get_default_conf_usdt
+from tests.conftest import EXMS, get_default_conf_usdt
 
 
 EXCHANGE_FIXTURE_TYPE = Tuple[Exchange, str]
@@ -309,7 +309,7 @@ def exchange(request, exchange_conf):
 
 @pytest.fixture(params=EXCHANGES, scope="class")
 def exchange_futures(request, exchange_conf, class_mocker):
-    if not EXCHANGES[request.param].get('futures') is True:
+    if EXCHANGES[request.param].get('futures') is not True:
         yield None, request.param
     else:
         exchange_conf = set_test_proxy(
@@ -322,13 +322,12 @@ def exchange_futures(request, exchange_conf, class_mocker):
 
         class_mocker.patch(
             'freqtrade.exchange.binance.Binance.fill_leverage_tiers')
-        class_mocker.patch('freqtrade.exchange.exchange.Exchange.fetch_trading_fees')
+        class_mocker.patch(f'{EXMS}.fetch_trading_fees')
         class_mocker.patch('freqtrade.exchange.okx.Okx.additional_exchange_init')
         class_mocker.patch('freqtrade.exchange.binance.Binance.additional_exchange_init')
         class_mocker.patch('freqtrade.exchange.bybit.Bybit.additional_exchange_init')
-        class_mocker.patch('freqtrade.exchange.exchange.Exchange.load_cached_leverage_tiers',
+        class_mocker.patch(f'{EXMS}.load_cached_leverage_tiers', return_value=None)
-                           return_value=None)
+        class_mocker.patch(f'{EXMS}.cache_leverage_tiers')
-        class_mocker.patch('freqtrade.exchange.exchange.Exchange.cache_leverage_tiers')
 
         exchange = ExchangeResolver.load_exchange(
             request.param, exchange_conf, validate=True, load_leverage_tiers=True)

tests/exchange/test_gate.py

@@ -7,18 +7,18 @@ from freqtrade.enums import MarginMode, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Gate
 from freqtrade.resolvers.exchange_resolver import ExchangeResolver
-from tests.conftest import get_patched_exchange
+from tests.conftest import EXMS, get_patched_exchange
 
 
 def test_validate_order_types_gate(default_conf, mocker):
     default_conf['exchange']['name'] = 'gate'
-    mocker.patch('freqtrade.exchange.Exchange._init_ccxt')
+    mocker.patch(f'{EXMS}._init_ccxt')
-    mocker.patch('freqtrade.exchange.Exchange._load_markets', return_value={})
+    mocker.patch(f'{EXMS}._load_markets', return_value={})
-    mocker.patch('freqtrade.exchange.Exchange.validate_pairs')
+    mocker.patch(f'{EXMS}.validate_pairs')
-    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
+    mocker.patch(f'{EXMS}.validate_timeframes')
-    mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
+    mocker.patch(f'{EXMS}.validate_stakecurrency')
-    mocker.patch('freqtrade.exchange.Exchange.validate_pricing')
+    mocker.patch(f'{EXMS}.validate_pricing')
-    mocker.patch('freqtrade.exchange.Exchange.name', 'Gate')
+    mocker.patch(f'{EXMS}.name', 'Gate')
     exch = ExchangeResolver.load_exchange('gate', default_conf, True)
     assert isinstance(exch, Gate)
 
@@ -105,7 +105,7 @@ def test_stoploss_adjust_gate(mocker, default_conf, sl1, sl2, sl3, side):
     ('maker', 0.0, 0.0),
 ])
 def test_fetch_my_trades_gate(mocker, default_conf, takerormaker, rate, cost):
-    mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True)
+    mocker.patch(f'{EXMS}.exchange_has', return_value=True)
     tick = {'ETH/USDT:USDT': {
         'info': {'user_id': '',
                  'taker_fee': '0.0018',

tests/exchange/test_huobi.py

@@ -5,7 +5,7 @@ import ccxt
 import pytest
 
 from freqtrade.exceptions import DependencyException, InvalidOrderException, OperationalException
-from tests.conftest import get_patched_exchange
+from tests.conftest import EXMS, get_patched_exchange
 from tests.exchange.test_exchange import ccxt_exceptionhandlers
 
 
@@ -26,8 +26,8 @@ def test_create_stoploss_order_huobi(default_conf, mocker, limitratio, expected,
         }
     })
     default_conf['dry_run'] = False
-    mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y: y)
 
     exchange = get_patched_exchange(mocker, default_conf, api_mock, 'huobi')
 
@@ -79,8 +79,8 @@ def test_create_stoploss_order_dry_run_huobi(default_conf, mocker):
     api_mock = MagicMock()
     order_type = 'stop-limit'
     default_conf['dry_run'] = True
-    mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y: y)
 
     exchange = get_patched_exchange(mocker, default_conf, api_mock, 'huobi')
 

tests/exchange/test_kraken.py

@@ -5,7 +5,7 @@ import ccxt
 import pytest
 
 from freqtrade.exceptions import DependencyException, InvalidOrderException
-from tests.conftest import get_patched_exchange
+from tests.conftest import EXMS, get_patched_exchange
 from tests.exchange.test_exchange import ccxt_exceptionhandlers
 
 
@@ -28,8 +28,8 @@ def test_buy_kraken_trading_agreement(default_conf, mocker):
     })
     default_conf['dry_run'] = False
 
-    mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y: y)
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken")
 
     order = exchange.create_order(
@@ -68,8 +68,8 @@ def test_sell_kraken_trading_agreement(default_conf, mocker):
     })
     default_conf['dry_run'] = False
 
-    mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y: y)
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken")
 
     order = exchange.create_order(pair='ETH/BTC', ordertype=order_type,
@@ -191,8 +191,8 @@ def test_create_stoploss_order_kraken(default_conf, mocker, ordertype, side, adj
     })
 
     default_conf['dry_run'] = False
-    mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y: y)
 
     exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken')
 
@@ -262,8 +262,8 @@ def test_create_stoploss_order_kraken(default_conf, mocker, ordertype, side, adj
 def test_create_stoploss_order_dry_run_kraken(default_conf, mocker, side):
     api_mock = MagicMock()
     default_conf['dry_run'] = True
-    mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y: y)
 
     exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken')
 

tests/exchange/test_kucoin.py

@@ -5,7 +5,7 @@ import ccxt
 import pytest
 
 from freqtrade.exceptions import DependencyException, InvalidOrderException, OperationalException
-from tests.conftest import get_patched_exchange
+from tests.conftest import EXMS, get_patched_exchange
 from tests.exchange.test_exchange import ccxt_exceptionhandlers
 
 
@@ -26,8 +26,8 @@ def test_create_stoploss_order_kucoin(default_conf, mocker, limitratio, expected
         }
     })
     default_conf['dry_run'] = False
-    mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y: y)
 
     exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kucoin')
     if order_type == 'limit':
@@ -87,8 +87,8 @@ def test_stoploss_order_dry_run_kucoin(default_conf, mocker):
     api_mock = MagicMock()
     order_type = 'market'
     default_conf['dry_run'] = True
-    mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y: y)
 
     exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kucoin')
 
@@ -125,3 +125,45 @@ def test_stoploss_adjust_kucoin(mocker, default_conf):
     # Test with invalid order case
     order['stopPrice'] = None
     assert exchange.stoploss_adjust(1501, order, 'sell')
+
+
+@pytest.mark.parametrize("side", ["buy", "sell"])
+@pytest.mark.parametrize("ordertype,rate", [
+    ("market", None),
+    ("market", 200),
+    ("limit", 200),
+    ("stop_loss_limit", 200)
+])
+def test_kucoin_create_order(default_conf, mocker, side, ordertype, rate):
+    api_mock = MagicMock()
+    order_id = 'test_prod_{}_{}'.format(side, randint(0, 10 ** 6))
+    api_mock.create_order = MagicMock(return_value={
+        'id': order_id,
+        'info': {
+            'foo': 'bar'
+        },
+        'symbol': 'XRP/USDT',
+        'amount': 1
+    })
+    default_conf['dry_run'] = False
+    mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y)
+    mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y: y)
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, id='kucoin')
+    exchange._set_leverage = MagicMock()
+    exchange.set_margin_mode = MagicMock()
+
+    order = exchange.create_order(
+        pair='XRP/USDT',
+        ordertype=ordertype,
+        side=side,
+        amount=1,
+        rate=rate,
+        leverage=1.0
+    )
+
+    assert 'id' in order
+    assert 'info' in order
+    assert order['id'] == order_id
+    assert order['amount'] == 1
+    # Status must be faked to open for kucoin.
+    assert order['status'] == 'open'

tests/exchange/test_okx.py

@@ -46,7 +46,7 @@ def test_get_maintenance_ratio_and_amt_okx(
     default_conf['margin_mode'] = 'isolated'
     default_conf['dry_run'] = False
     mocker.patch.multiple(
-        'freqtrade.exchange.Okx',
+        'freqtrade.exchange.okx.Okx',
         exchange_has=MagicMock(return_value=True),
         load_leverage_tiers=MagicMock(return_value={
             'ETH/USDT:USDT': [

tests/freqai/conftest.py

@@ -27,7 +27,7 @@ def freqai_conf(default_conf, tmpdir):
             "timerange": "20180110-20180115",
             "freqai": {
                 "enabled": True,
-                "purge_old_models": True,
+                "purge_old_models": 2,
                 "train_period_days": 2,
                 "backtest_period_days": 10,
                 "live_retrain_hours": 0,
@@ -46,6 +46,8 @@ def freqai_conf(default_conf, tmpdir):
                     "use_SVM_to_remove_outliers": True,
                     "stratify_training_data": 0,
                     "indicator_periods_candles": [10],
+                    "shuffle_after_split": False,
+                    "buffer_train_data_candles": 0
                 },
                 "data_split_parameters": {"test_size": 0.33, "shuffle": False},
                 "model_training_parameters": {"n_estimators": 100},

tests/freqai/test_freqai_backtesting.py

@@ -35,8 +35,8 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, c
     args = get_args(args)
     bt_config = setup_optimize_configuration(args, RunMode.BACKTEST)
     Backtesting(bt_config)
-    assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical '
+    assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical',
-                      'populate_any_indicators.', caplog)
+                      caplog)
     Backtesting.cleanup()