Merge pull request #5491 from freqtrade/new_release

New release 2021.8

.devcontainer/devcontainer.json

@@ -1,11 +1,20 @@
 {
     "name": "freqtrade Develop",
-
-    "dockerComposeFile": [
-        "docker-compose.yml"
+    "build": {
+        "dockerfile": "Dockerfile",
+        "context": ".."
+    },
+    // Use 'forwardPorts' to make a list of ports inside the container available locally.
+    "forwardPorts": [
+        8080
     ],
+    "mounts": [
+        "source=freqtrade-bashhistory,target=/home/ftuser/commandhistory,type=volume"
+    ],
+    // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root.
+    "remoteUser": "ftuser",
 
-    "service": "ft_vscode",
+    "postCreateCommand": "freqtrade create-userdir --userdir user_data/",
 
     "workspaceFolder": "/freqtrade/",
 
@@ -25,20 +34,6 @@
         "ms-python.vscode-pylance",
         "davidanson.vscode-markdownlint",
         "ms-azuretools.vscode-docker",
+        "vscode-icons-team.vscode-icons",
     ],
-
-    // Use 'forwardPorts' to make a list of ports inside the container available locally.
-    // "forwardPorts": [],
-
-    // Uncomment the next line if you want start specific services in your Docker Compose config.
-    // "runServices": [],
-
-    // Uncomment the next line if you want to keep your containers running after VS Code shuts down.
-    // "shutdownAction": "none",
-
-    // Uncomment the next line to run commands after the container is created - for example installing curl.
-    // "postCreateCommand": "sudo apt-get update && apt-get install -y git",
-
-    // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root.
-    "remoteUser": "ftuser"
 }

.devcontainer/docker-compose.yml

@@ -1,24 +0,0 @@
----
-version: '3'
-services:
-  ft_vscode:
-    build:
-      context: ..
-      dockerfile: ".devcontainer/Dockerfile"
-    volumes:
-      # Allow git usage within container
-      - "${HOME}/.ssh:/home/ftuser/.ssh:ro"
-      - "${HOME}/.gitconfig:/home/ftuser/.gitconfig:ro"
-      - ..:/freqtrade:cached
-      # Persist bash-history
-      - freqtrade-vscode-server:/home/ftuser/.vscode-server
-      - freqtrade-bashhistory:/home/ftuser/commandhistory
-    # Expose API port
-    ports:
-      - "127.0.0.1:8080:8080"
-    command: /bin/sh -c "while sleep 1000; do :; done"
-
-
-volumes:
-  freqtrade-vscode-server:
-  freqtrade-bashhistory:

.github/workflows/ci.yml

@@ -79,13 +79,13 @@ jobs:
 
     - name: Backtesting
       run: |
-        cp config_bittrex.json.example config.json
+        cp config_examples/config_bittrex.example.json config.json
         freqtrade create-userdir --userdir user_data
         freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy
 
     - name: Hyperopt
       run: |
-        cp config_bittrex.json.example config.json
+        cp config_examples/config_bittrex.example.json config.json
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
@@ -172,13 +172,13 @@ jobs:
 
     - name: Backtesting
       run: |
-        cp config_bittrex.json.example config.json
+        cp config_examples/config_bittrex.example.json config.json
         freqtrade create-userdir --userdir user_data
         freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy
 
     - name: Hyperopt
       run: |
-        cp config_bittrex.json.example config.json
+        cp config_examples/config_bittrex.example.json config.json
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
@@ -239,13 +239,13 @@ jobs:
 
     - name: Backtesting
       run: |
-        cp config_bittrex.json.example config.json
+        cp config_examples/config_bittrex.example.json config.json
         freqtrade create-userdir --userdir user_data
         freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy
 
     - name: Hyperopt
       run: |
-        cp config_bittrex.json.example config.json
+        cp config_examples/config_bittrex.example.json config.json
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
@@ -334,6 +334,7 @@ jobs:
     runs-on: ubuntu-20.04
 
     if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade'
+
     steps:
     - uses: actions/checkout@v2
 
@@ -411,3 +412,31 @@ jobs:
         channel: '#notifications'
         url: ${{ secrets.SLACK_WEBHOOK }}
 
+
+  deploy_arm:
+    needs: [ deploy ]
+    # Only run on 64bit machines
+    runs-on: [self-hosted, linux, ARM64]
+    if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade'
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Extract branch name
+      shell: bash
+      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF##*/})"
+      id: extract_branch
+
+    - name: Dockerhub login
+      env:
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      run: |
+        echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+
+    - name: Build and test and push docker images
+      env:
+        IMAGE_NAME: freqtradeorg/freqtrade
+        BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
+      run: |
+        build_helpers/publish_docker_arm64.sh

.gitignore

@@ -95,3 +95,8 @@ target/
 
 #exceptions
 !*.gitkeep
+!config_examples/config_binance.example.json
+!config_examples/config_bittrex.example.json
+!config_examples/config_ftx.example.json
+!config_examples/config_full.example.json
+!config_examples/config_kraken.example.json

.travis.yml

@@ -26,12 +26,12 @@ jobs:
       # - coveralls || true
       name: pytest
     - script:
-      - cp config_bittrex.json.example config.json
+      - cp config_examples/config_bittrex.example.json config.json
       - freqtrade create-userdir --userdir user_data
       - freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy
       name: backtest
     - script:
-      - cp config_bittrex.json.example config.json
+      - cp config_examples/config_bittrex.example.json config.json
       - freqtrade create-userdir --userdir user_data
       - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily
       name: hyperopt

CONTRIBUTING.md

@@ -12,7 +12,7 @@ Few pointers for contributions:
 - New features need to contain unit tests, must conform to PEP8 (max-line-length = 100) and should be documented with the introduction PR.
 - PR's can be declared as `[WIP]` - which signify Work in Progress Pull Requests (which are not finished).
 
-If you are unsure, discuss the feature on our [discord server](https://discord.gg/p7nuUNVfP7), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR.
+If you are unsure, discuss the feature on our [discord server](https://discord.gg/p7nuUNVfP7) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a Pull Request.
 
 ## Getting started
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9.5-slim-buster as base
+FROM python:3.9.6-slim-buster as base
 
 # Setup env
 ENV LANG C.UTF-8

README.md

@@ -26,8 +26,8 @@ hesitate to read the source code and understand the mechanism of this bot.
 
 Please read the [exchange specific notes](docs/exchanges.md) to learn about eventual, special configurations needed for each exchange.
 
+- [X] [Binance](https://www.binance.com/) ([*Note for binance users](docs/exchanges.md#binance-blacklist))
 - [X] [Bittrex](https://bittrex.com/)
-- [X] [Binance](https://www.binance.com/) ([*Note for binance users](docs/exchanges.md#blacklists))
 - [X] [Kraken](https://kraken.com/)
 - [X] [FTX](https://ftx.com)
 - [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_
@@ -37,6 +37,7 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even
 Exchanges confirmed working by the community:
 
 - [X] [Bitvavo](https://bitvavo.com/)
+- [X] [Kucoin](https://www.kucoin.com/)
 
 ## Documentation
 
@@ -141,13 +142,9 @@ The project is currently setup in two main branches:
 
 ## Support
 
-### Help / Discord / Slack
+### Help / Discord
 
-For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join our slack channel.
-
-Please check out our [discord server](https://discord.gg/p7nuUNVfP7).
-
-You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw).
+For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join the Freqtrade [discord server](https://discord.gg/p7nuUNVfP7).
 
 ### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue)
 
@@ -178,7 +175,7 @@ to understand the requirements before sending your pull-requests.
 Coding is not a necessity to contribute - maybe start with improving our documentation?
 Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/good%20first%20issue) can be good first contributions, and will help get you familiar with the codebase.
 
-**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/p7nuUNVfP7) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it.
+**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/p7nuUNVfP7) (please use the #dev channel for this). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it.
 
 **Important:** Always create your PR against the `develop` branch, not `stable`.
 

build_helpers/install_windows.ps1

@@ -6,10 +6,13 @@ python -m pip install --upgrade pip
 $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"
 
 if ($pyv -eq '3.7') {
-    pip install build_helpers\TA_Lib-0.4.20-cp37-cp37m-win_amd64.whl
+    pip install build_helpers\TA_Lib-0.4.21-cp37-cp37m-win_amd64.whl
 }
 if ($pyv -eq '3.8') {
-    pip install build_helpers\TA_Lib-0.4.20-cp38-cp38-win_amd64.whl
+    pip install build_helpers\TA_Lib-0.4.21-cp38-cp38-win_amd64.whl
+}
+if ($pyv -eq '3.9') {
+    pip install build_helpers\TA_Lib-0.4.21-cp39-cp39-win_amd64.whl
 }
 
 pip install -r requirements-dev.txt

build_helpers/publish_docker_arm64.sh

@@ -0,0 +1,78 @@
+#!/bin/sh
+
+# Use BuildKit, otherwise building on ARM fails
+export DOCKER_BUILDKIT=1
+
+# Replace / with _ to create a valid tag
+TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
+TAG_PLOT=${TAG}_plot
+TAG_PI="${TAG}_pi"
+
+TAG_ARM=${TAG}_arm
+TAG_PLOT_ARM=${TAG_PLOT}_arm
+CACHE_IMAGE=freqtradeorg/freqtrade_cache
+
+echo "Running for ${TAG}"
+
+# Add commit and commit_message to docker container
+echo "${GITHUB_SHA}" > freqtrade_commit
+
+if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then
+    echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache"
+    # Build regular image
+    docker build -t freqtrade:${TAG_ARM} .
+
+else
+    echo "event ${GITHUB_EVENT_NAME}: building with cache"
+    # Build regular image
+    docker pull ${IMAGE_NAME}:${TAG_ARM}
+    docker build --cache-from ${IMAGE_NAME}:${TAG_ARM} -t freqtrade:${TAG_ARM} .
+
+fi
+
+if [ $? -ne 0 ]; then
+    echo "failed building multiarch images"
+    return 1
+fi
+# Tag image for upload and next build step
+docker tag freqtrade:$TAG_ARM ${CACHE_IMAGE}:$TAG_ARM
+
+docker build --cache-from freqtrade:${TAG_ARM} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_PLOT_ARM} -f docker/Dockerfile.plot .
+
+docker tag freqtrade:$TAG_PLOT_ARM ${CACHE_IMAGE}:$TAG_PLOT_ARM
+
+# Run backtest
+docker run --rm -v $(pwd)/config_examples/config_bittrex.example.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG_ARM} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV2
+
+if [ $? -ne 0 ]; then
+    echo "failed running backtest"
+    return 1
+fi
+
+docker images
+
+# docker push ${IMAGE_NAME}
+docker push ${CACHE_IMAGE}:$TAG_PLOT_ARM
+docker push ${CACHE_IMAGE}:$TAG_ARM
+
+# Create multi-arch image
+# Make sure that all images contained here are pushed to github first.
+# Otherwise installation might fail.
+echo "create manifests"
+
+docker manifest create --amend ${IMAGE_NAME}:${TAG} ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} ${CACHE_IMAGE}:${TAG}
+docker manifest push -p ${IMAGE_NAME}:${TAG}
+
+docker manifest create ${IMAGE_NAME}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT_ARM} ${CACHE_IMAGE}:${TAG_PLOT}
+docker manifest push -p ${IMAGE_NAME}:${TAG_PLOT}
+
+# Tag as latest for develop builds
+if [ "${TAG}" = "develop" ]; then
+    docker manifest create ${IMAGE_NAME}:latest ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} ${CACHE_IMAGE}:${TAG}
+    docker manifest push -p ${IMAGE_NAME}:latest
+fi
+
+docker images
+
+# Cleanup old images from arm64 node.
+docker image prune -a --force --filter "until=24h"

build_helpers/publish_docker_multi.sh

@@ -9,7 +9,8 @@ TAG_PI="${TAG}_pi"
 
 PI_PLATFORM="linux/arm/v7"
 echo "Running for ${TAG}"
-CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache
+CACHE_IMAGE=freqtradeorg/freqtrade_cache
+CACHE_TAG=${CACHE_IMAGE}:${TAG_PI}_cache
 
 # Add commit and commit_message to docker container
 echo "${GITHUB_SHA}" > freqtrade_commit
@@ -45,14 +46,14 @@ if [ $? -ne 0 ]; then
     return 1
 fi
 # Tag image for upload and next build step
-docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG
+docker tag freqtrade:$TAG ${CACHE_IMAGE}:$TAG
 
-docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot .
+docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot .
 
-docker tag freqtrade:$TAG_PLOT ${IMAGE_NAME}:$TAG_PLOT
+docker tag freqtrade:$TAG_PLOT ${CACHE_IMAGE}:$TAG_PLOT
 
 # Run backtest
-docker run --rm -v $(pwd)/config_bittrex.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy
+docker run --rm -v $(pwd)/config_examples/config_bittrex.example.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV2
 
 if [ $? -ne 0 ]; then
     echo "failed running backtest"
@@ -61,22 +62,9 @@ fi
 
 docker images
 
-docker push ${IMAGE_NAME}
-docker push ${IMAGE_NAME}:$TAG_PLOT
-docker push ${IMAGE_NAME}:$TAG
-
-# Create multiarch image
-# Make sure that all images contained here are pushed to github first.
-# Otherwise installation might fail.
-
-docker manifest create freqtradeorg/freqtrade:${TAG} ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI}
-docker manifest push freqtradeorg/freqtrade:${TAG}
-
-# Tag as latest for develop builds
-if [ "${TAG}" = "develop" ]; then
-    docker manifest create freqtradeorg/freqtrade:latest ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI}
-    docker manifest push freqtradeorg/freqtrade:latest
-fi
+docker push ${CACHE_IMAGE}
+docker push ${CACHE_IMAGE}:$TAG_PLOT
+docker push ${CACHE_IMAGE}:$TAG
 
 
 docker images

config_examples/config_binance.example.json

@@ -13,7 +13,7 @@
     },
     "bid_strategy": {
         "ask_last_balance": 0.0,
-        "use_order_book": false,
+        "use_order_book": true,
         "order_book_top": 1,
         "check_depth_of_market": {
             "enabled": false,
@@ -21,12 +21,8 @@
         }
     },
     "ask_strategy": {
-        "use_order_book": false,
-        "order_book_min": 1,
-        "order_book_max": 1,
-        "use_sell_signal": true,
-        "sell_profit_only": false,
-        "ignore_roi_if_buy_signal": false
+        "use_order_book": true,
+        "order_book_top": 1
     },
     "exchange": {
         "name": "binance",

config_examples/config_bittrex.example.json

@@ -12,7 +12,7 @@
         "sell": 30
     },
     "bid_strategy": {
-        "use_order_book": false,
+        "use_order_book": true,
         "ask_last_balance": 0.0,
         "order_book_top": 1,
         "check_depth_of_market": {
@@ -21,12 +21,8 @@
         }
     },
     "ask_strategy":{
-        "use_order_book": false,
-        "order_book_min": 1,
-        "order_book_max": 1,
-        "use_sell_signal": true,
-        "sell_profit_only": false,
-        "ignore_roi_if_buy_signal": false
+        "use_order_book": true,
+        "order_book_top": 1
     },
     "exchange": {
         "name": "bittrex",

config_examples/config_ftx.example.json

@@ -13,7 +13,7 @@
     },
     "bid_strategy": {
         "ask_last_balance": 0.0,
-        "use_order_book": false,
+        "use_order_book": true,
         "order_book_top": 1,
         "check_depth_of_market": {
             "enabled": false,
@@ -21,12 +21,8 @@
         }
     },
     "ask_strategy": {
-        "use_order_book": false,
-        "order_book_min": 1,
-        "order_book_max": 1,
-        "use_sell_signal": true,
-        "sell_profit_only": false,
-        "ignore_roi_if_buy_signal": false
+        "use_order_book": true,
+        "order_book_top": 1
     },
     "exchange": {
         "name": "ftx",

config_examples/config_full.example.json

@@ -14,6 +14,10 @@
     "trailing_stop_positive": 0.005,
     "trailing_stop_positive_offset": 0.0051,
     "trailing_only_offset_is_reached": false,
+    "use_sell_signal": true,
+    "sell_profit_only": false,
+    "sell_profit_offset": 0.0,
+    "ignore_roi_if_buy_signal": false,
     "minimal_roi": {
         "40":  0.0,
         "30":  0.01,
@@ -28,7 +32,7 @@
     },
     "bid_strategy": {
         "price_side": "bid",
-        "use_order_book": false,
+        "use_order_book": true,
         "ask_last_balance": 0.0,
         "order_book_top": 1,
         "check_depth_of_market": {
@@ -38,13 +42,8 @@
     },
     "ask_strategy":{
         "price_side": "ask",
-        "use_order_book": false,
-        "order_book_min": 1,
-        "order_book_max": 1,
-        "use_sell_signal": true,
-        "sell_profit_only": false,
-        "sell_profit_offset": 0.0,
-        "ignore_roi_if_buy_signal": false
+        "use_order_book": true,
+        "order_book_top": 1
     },
     "order_types": {
         "buy": "limit",
@@ -79,33 +78,6 @@
             "refresh_period": 1440
         }
     ],
-    "protections": [
-        {
-            "method": "StoplossGuard",
-            "lookback_period_candles": 60,
-            "trade_limit": 4,
-            "stop_duration_candles": 60,
-            "only_per_pair": false
-        },
-        {
-            "method": "CooldownPeriod",
-            "stop_duration_candles": 20
-        },
-        {
-            "method": "MaxDrawdown",
-            "lookback_period_candles": 200,
-            "trade_limit": 20,
-            "stop_duration_candles": 10,
-            "max_allowed_drawdown": 0.2
-        },
-        {
-            "method": "LowProfitPairs",
-            "lookback_period_candles": 360,
-            "trade_limit": 1,
-            "stop_duration_candles": 2,
-            "required_profit": 0.02
-        }
-    ],
     "exchange": {
         "name": "binance",
         "sandbox": false,
@@ -202,7 +174,7 @@
         "heartbeat_interval": 60
     },
     "disable_dataframe_checks": false,
-    "strategy": "DefaultStrategy",
+    "strategy": "SampleStrategy",
     "strategy_path": "user_data/strategies/",
     "dataformat_ohlcv": "json",
     "dataformat_trades": "jsongz"

config_examples/config_kraken.example.json

@@ -12,7 +12,7 @@
         "sell": 30
     },
     "bid_strategy": {
-        "use_order_book": false,
+        "use_order_book": true,
         "ask_last_balance": 0.0,
         "order_book_top": 1,
         "check_depth_of_market": {
@@ -21,12 +21,8 @@
         }
     },
     "ask_strategy":{
-        "use_order_book": false,
-        "order_book_min": 1,
-        "order_book_max": 1,
-        "use_sell_signal": true,
-        "sell_profit_only": false,
-        "ignore_roi_if_buy_signal": false
+        "use_order_book": true,
+        "order_book_top": 1
     },
     "exchange": {
         "name": "kraken",

docker/Dockerfile.plot

@@ -1,5 +1,6 @@
-ARG sourceimage=develop
-FROM freqtradeorg/freqtrade:${sourceimage}
+ARG sourceimage=freqtradeorg/freqtrade
+ARG sourcetag=develop
+FROM ${sourceimage}:${sourcetag}
 
 # Install dependencies
 COPY requirements-plot.txt /freqtrade/

docs/advanced-hyperopt.md

@@ -32,6 +32,7 @@ class SuperDuperHyperOptLoss(IHyperOptLoss):
     def hyperopt_loss_function(results: DataFrame, trade_count: int,
                                min_date: datetime, max_date: datetime,
                                config: Dict, processed: Dict[str, DataFrame],
+                               backtest_stats: Dict[str, Any],
                                *args, **kwargs) -> float:
         """
         Objective function, returns smaller number for better results
@@ -53,7 +54,7 @@ class SuperDuperHyperOptLoss(IHyperOptLoss):
 
 Currently, the arguments are:
 
-* `results`: DataFrame containing the result  
+* `results`: DataFrame containing the resulting trades.
     The following columns are available in results (corresponds to the output-file of backtesting when used with `--export trades`):  
     `pair, profit_ratio, profit_abs, open_date, open_rate, fee_open, close_date, close_rate, fee_close, amount, trade_duration, is_open, sell_reason, stake_amount, min_rate, max_rate, stop_loss_ratio, stop_loss_abs`
 * `trade_count`: Amount of trades (identical to `len(results)`)
@@ -61,6 +62,7 @@ Currently, the arguments are:
 * `min_date`: End date of the timerange used
 * `config`: Config object used (Note: Not all strategy-related parameters will be updated here if they are part of a hyperopt space).
 * `processed`: Dict of Dataframes with the pair as keys containing the data used for backtesting.
+* `backtest_stats`: Backtesting statistics using the same format as the backtesting file "strategy" substructure. Available fields can be seen in `generate_strategy_stats()` in `optimize_reports.py`.
 
 This function needs to return a floating point number (`float`). Smaller numbers will be interpreted as better results. The parameters and balancing for this is up to you.
 

docs/backtesting.md

@@ -62,7 +62,7 @@ optional arguments:
                         this together with `--export trades`, the strategy-
                         name is injected into the filename (so `backtest-
                         data.json` becomes `backtest-data-
-                        DefaultStrategy.json`
+                        SampleStrategy.json`
   --export {none,trades}
                         Export backtest results (default: trades).
   --export-filename PATH
@@ -302,7 +302,6 @@ A backtesting result will look like that:
 | Days win/draw/lose    | 12 / 82 / 25        |
 | Avg. Duration Winners | 4:23:00             |
 | Avg. Duration Loser   | 6:55:00             |
-| Zero Duration Trades  | 4.6% (20)           |
 | Rejected Buy signals  | 3089                |
 |                       |                     |
 | Min balance           | 0.00945123 BTC      |
@@ -390,7 +389,6 @@ It contains some useful key metrics about performance of your strategy on backte
 | Days win/draw/lose    | 12 / 82 / 25        |
 | Avg. Duration Winners | 4:23:00             |
 | Avg. Duration Loser   | 6:55:00             |
-| Zero Duration Trades  | 4.6% (20)           |
 | Rejected Buy signals  | 3089                |
 |                       |                     |
 | Min balance           | 0.00945123 BTC      |
@@ -420,7 +418,6 @@ It contains some useful key metrics about performance of your strategy on backte
 - `Best day` / `Worst day`: Best and worst day based on daily profit.
 - `Days win/draw/lose`: Winning / Losing days (draws are usually days without closed trade).
 - `Avg. Duration Winners` / `Avg. Duration Loser`: Average durations for winning and losing trades.
-- `Zero Duration Trades`: A number of trades that completed within same candle as they opened and had `trailing_stop_loss` sell reason. A significant amount of such trades may indicate that strategy is exploiting trailing stoploss behavior in backtesting and produces unrealistic results.
 - `Rejected Buy signals`: Buy signals that could not be acted upon due to max_open_trades being reached.
 - `Min balance` / `Max balance`: Lowest and Highest Wallet balance during the backtest period.
 - `Drawdown`: Maximum drawdown experienced. For example, the value of 50% means that from highest to subsequent lowest point, a 50% drop was experienced).

docs/bot-basics.md

@@ -35,12 +35,13 @@ By default, loop runs every few seconds (`internals.process_throttle_secs`) and
   * Calls `check_buy_timeout()` strategy callback for open buy orders.
   * Calls `check_sell_timeout()` strategy callback for open sell orders.
 * Verifies existing positions and eventually places sell orders.
-  * Considers stoploss, ROI and sell-signal.
-  * Determine sell-price based on `ask_strategy` configuration setting.
+  * Considers stoploss, ROI and sell-signal, `custom_sell()` and `custom_stoploss()`.
+  * Determine sell-price based on `ask_strategy` configuration setting or by using the `custom_exit_price()` callback.
   * Before a sell order is placed, `confirm_trade_exit()` strategy callback is called.
 * Check if trade-slots are still available (if `max_open_trades` is reached).
 * Verifies buy signal trying to enter new positions.
-  * Determine buy-price based on `bid_strategy` configuration setting.
+  * Determine buy-price based on `bid_strategy` configuration setting, or by using the `custom_entry_price()` callback.
+  * Determine stake size by calling the `custom_stake_amount()` callback.
   * Before a buy order is placed, `confirm_trade_entry()` strategy callback is called.
 
 This loop will be repeated again and again until the bot is stopped.
@@ -52,9 +53,10 @@ This loop will be repeated again and again until the bot is stopped.
 * Load historic data for configured pairlist.
 * Calls `bot_loop_start()` once.
 * Calculate indicators (calls `populate_indicators()` once per pair).
-* Calculate buy / sell signals (calls `populate_buy_trend()` and `populate_sell_trend()` once per pair)
-* Confirm trade buy / sell (calls `confirm_trade_entry()` and `confirm_trade_exit()` if implemented in the strategy)
+* Calculate buy / sell signals (calls `populate_buy_trend()` and `populate_sell_trend()` once per pair).
 * Loops per candle simulating entry and exit points.
+  * Confirm trade buy / sell (calls `confirm_trade_entry()` and `confirm_trade_exit()` if implemented in the strategy).
+  * Call `custom_stoploss()` and `custom_sell()` to find custom exit points.
 * Generate backtest report output
 
 !!! Note

docs/configuration.md

@@ -5,11 +5,42 @@ By default, these settings are configured via the configuration file (see below)
 
 ## The Freqtrade configuration file
 
-The bot uses a set of configuration parameters during its operation that all together conform the bot configuration. It normally reads its configuration from a file (Freqtrade configuration file).
+The bot uses a set of configuration parameters during its operation that all together conform to the bot configuration. It normally reads its configuration from a file (Freqtrade configuration file).
 
 Per default, the bot loads the configuration from the `config.json` file, located in the current working directory.
 
-You can specify a different configuration file used by the bot with the `-c/--config` command line option.
+You can specify a different configuration file used by the bot with the `-c/--config` command-line option.
+
+If you used the [Quick start](installation.md/#quick-start) method for installing 
+the bot, the installation script should have already created the default configuration file (`config.json`) for you.
+
+If the default configuration file is not created we recommend to use `freqtrade new-config --config config.json` to generate a basic configuration file.
+
+The Freqtrade configuration file is to be written in JSON format.
+
+Additionally to the standard JSON syntax, you may use one-line `// ...` and multi-line `/* ... */` comments in your configuration files and trailing commas in the lists of parameters.
+
+Do not worry if you are not familiar with JSON format -- simply open the configuration file with an editor of your choice, make some changes to the parameters you need, save your changes and, finally, restart the bot or, if it was previously stopped, run it again with the changes you made to the configuration. The bot validates the syntax of the configuration file at startup and will warn you if you made any errors editing it, pointing out problematic lines.
+
+### Environment variables
+
+Set options in the Freqtrade configuration via environment variables.
+This takes priority over the corresponding value in configuration or strategy.
+
+Environment variables must be prefixed with  `FREQTRADE__` to be loaded to the freqtrade configuration.
+
+`__` serves as level separator, so the format used should correspond to `FREQTRADE__{section}__{key}`.
+As such - an environment variable defined as  `export FREQTRADE__STAKE_AMOUNT=200` would result in `{stake_amount: 200}`.
+
+A more complex example might be `export FREQTRADE__EXCHANGE__KEY=<yourExchangeKey>` to keep your exchange key secret. This will move the value to the `exchange.key` section of the configuration.
+Using this scheme, all configuration settings will also be available as environment variables.
+
+Please note that Environment variables will overwrite corresponding settings in your configuration, but command line Arguments will always win.
+
+!!! Note
+    Environment variables detected are logged at startup - so if you can't find why a value is not what you think it should be based on the configuration, make sure it's not loaded from an environment variable.
+
+### Multiple configuration files
 
 Multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream.
 
@@ -22,36 +53,27 @@ Multiple configuration files can be specified and used by the bot or the bot can
     The 2nd file should only specify what you intend to override.
     If a key is in more than one of the configurations, then the "last specified configuration" wins (in the above example, `config-private.json`).
 
-If you used the [Quick start](installation.md/#quick-start) method for installing 
-the bot, the installation script should have already created the default configuration file (`config.json`) for you.
-
-If default configuration file is not created we recommend you to use `freqtrade new-config --config config.json` to generate a basic configuration file.
-
-The Freqtrade configuration file is to be written in the JSON format.
-
-Additionally to the standard JSON syntax, you may use one-line `// ...` and multi-line `/* ... */` comments in your configuration files and trailing commas in the lists of parameters.
-
-Do not worry if you are not familiar with JSON format -- simply open the configuration file with an editor of your choice, make some changes to the parameters you need, save your changes and, finally, restart the bot or, if it was previously stopped, run it again with the changes you made to the configuration. The bot validates syntax of the configuration file at startup and will warn you if you made any errors editing it, pointing out problematic lines.
-
 ## Configuration parameters
 
 The table below will list all configuration parameters available.
 
 Freqtrade can also load many options via command line (CLI) arguments (check out the commands `--help` output for details).
-The prevelance for all Options is as follows:
+The prevalence for all Options is as follows:
 
 - CLI arguments override any other option
-- Configuration files are used in sequence (last file wins), and override Strategy configurations.
-- Strategy configurations are only used if they are not set via configuration or via command line arguments. These options are marked with [Strategy Override](#parameters-in-the-strategy) in the below table.
+- [Environment Variables](#environment-variables)
+- Configuration files are used in sequence (the last file wins) and override Strategy configurations.
+- Strategy configurations are only used if they are not set via configuration or command-line arguments. These options are marked with [Strategy Override](#parameters-in-the-strategy) in the below table.
 
 Mandatory parameters are marked as **Required**, which means that they are required to be set in one of the possible ways.
 
 |  Parameter | Description |
 |------------|-------------|
-| `max_open_trades` | **Required.** Number of open trades your bot is allowed to have. Only one open trade per pair is possible, so the length of your pairlist is another limitation which can apply. If -1 then it is ignored (i.e. potentially unlimited open trades, limited by the pairlist). [More information below](#configuring-amount-per-trade).<br> **Datatype:** Positive integer or -1.
+| `max_open_trades` | **Required.** Number of open trades your bot is allowed to have. Only one open trade per pair is possible, so the length of your pairlist is another limitation that can apply. If -1 then it is ignored (i.e. potentially unlimited open trades, limited by the pairlist). [More information below](#configuring-amount-per-trade).<br> **Datatype:** Positive integer or -1.
 | `stake_currency` | **Required.** Crypto-currency used for trading. <br> **Datatype:** String
 | `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#configuring-amount-per-trade). <br> **Datatype:** Positive float or `"unlimited"`.
 | `tradable_balance_ratio` | Ratio of the total account balance the bot is allowed to trade. [More information below](#configuring-amount-per-trade). <br>*Defaults to `0.99` 99%).*<br> **Datatype:** Positive float between `0.1` and `1.0`.
+| `available_capital` | Available starting capital for the bot. Useful when running multiple bots on the same exchange account.[More information below](#configuring-amount-per-trade). <br> **Datatype:** Positive float.
 | `amend_last_stake_amount` | Use reduced last stake amount if necessary. [More information below](#configuring-amount-per-trade). <br>*Defaults to `false`.* <br> **Datatype:** Boolean
 | `last_stake_amount_min_ratio` | Defines minimum stake amount that has to be left and executed. Applies only to the last stake amount when it's amended to a reduced value (i.e. if `amend_last_stake_amount` is set to `true`). [More information below](#configuring-amount-per-trade). <br>*Defaults to `0.5`.* <br> **Datatype:** Float (as ratio)
 | `amount_reserve_percent` | Reserve some amount in min pair stake amount. The bot will reserve `amount_reserve_percent` + stoploss value when calculating min pair stake amount in order to avoid possible trade refusals. <br>*Defaults to `0.05` (5%).* <br> **Datatype:** Positive Float as ratio.
@@ -74,21 +96,21 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `bid_strategy.price_side` | Select the side of the spread the bot should look at to get the buy rate. [More information below](#buy-price-side).<br> *Defaults to `bid`.* <br> **Datatype:** String (either `ask` or `bid`).
 | `bid_strategy.ask_last_balance` | **Required.** Interpolate the bidding price. More information [below](#buy-price-without-orderbook-enabled).
 | `bid_strategy.use_order_book` | Enable buying using the rates in [Order Book Bids](#buy-price-with-orderbook-enabled). <br> **Datatype:** Boolean
-| `bid_strategy.order_book_top` | Bot will use the top N rate in Order Book Bids to buy. I.e. a value of 2 will allow the bot to pick the 2nd bid rate in [Order Book Bids](#buy-price-with-orderbook-enabled). <br>*Defaults to `1`.*  <br> **Datatype:** Positive Integer
+| `bid_strategy.order_book_top` | Bot will use the top N rate in Order Book "price_side" to buy. I.e. a value of 2 will allow the bot to pick the 2nd bid rate in [Order Book Bids](#buy-price-with-orderbook-enabled). <br>*Defaults to `1`.*  <br> **Datatype:** Positive Integer
 | `bid_strategy. check_depth_of_market.enabled` | Do not buy if the difference of buy orders and sell orders is met in Order Book. [Check market depth](#check-depth-of-market). <br>*Defaults to `false`.* <br> **Datatype:** Boolean
 | `bid_strategy. check_depth_of_market.bids_to_ask_delta` | The difference ratio of buy orders and sell orders found in Order Book. A value below 1 means sell order size is greater, while value greater than 1 means buy order size is higher. [Check market depth](#check-depth-of-market) <br> *Defaults to `0`.*  <br> **Datatype:** Float (as ratio)
 | `ask_strategy.price_side` | Select the side of the spread the bot should look at to get the sell rate. [More information below](#sell-price-side).<br> *Defaults to `ask`.* <br> **Datatype:** String (either `ask` or `bid`).
 | `ask_strategy.bid_last_balance` | Interpolate the selling price. More information [below](#sell-price-without-orderbook-enabled).
 | `ask_strategy.use_order_book` | Enable selling of open trades using [Order Book Asks](#sell-price-with-orderbook-enabled). <br> **Datatype:** Boolean
-| `ask_strategy.order_book_min` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. <br>*Defaults to `1`.* <br> **Datatype:** Positive Integer
-| `ask_strategy.order_book_max` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. <br>*Defaults to `1`.* <br> **Datatype:** Positive Integer
-| `ask_strategy.use_sell_signal` | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `true`.* <br> **Datatype:** Boolean
-| `ask_strategy.sell_profit_only` | Wait until the bot reaches `ask_strategy.sell_profit_offset` before taking a sell decision. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.* <br> **Datatype:** Boolean
-| `ask_strategy.sell_profit_offset` | Sell-signal is only active above this value. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `0.0`.* <br> **Datatype:** Float (as ratio)
-| `ask_strategy.ignore_roi_if_buy_signal` | Do not sell if the buy signal is still active. This setting takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.* <br> **Datatype:** Boolean
-| `ask_strategy.ignore_buying_expired_candle_after` | Specifies the number of seconds until a buy signal is no longer used. <br> **Datatype:** Integer
+| `ask_strategy.order_book_top` | Bot will use the top N rate in Order Book "price_side" to sell. I.e. a value of 2 will allow the bot to pick the 2nd ask rate in [Order Book Asks](#sell-price-with-orderbook-enabled)<br>*Defaults to `1`.* <br> **Datatype:** Positive Integer
+| `use_sell_signal` | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `true`.* <br> **Datatype:** Boolean
+| `sell_profit_only` | Wait until the bot reaches `sell_profit_offset` before taking a sell decision. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.* <br> **Datatype:** Boolean
+| `sell_profit_offset` | Sell-signal is only active above this value. Only active in combination with `sell_profit_only=True`. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `0.0`.* <br> **Datatype:** Float (as ratio)
+| `ignore_roi_if_buy_signal` | Do not sell if the buy signal is still active. This setting takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.* <br> **Datatype:** Boolean
+| `ignore_buying_expired_candle_after` | Specifies the number of seconds until a buy signal is no longer used. <br> **Datatype:** Integer
 | `order_types` | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Dict
 | `order_time_in_force` | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Dict
+| `custom_price_max_distance_ratio` | Configure maximum distance ratio between current and custom entry or exit price. <br>*Defaults to `0.02` 2%).*<br> **Datatype:** Positive float
 | `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). <br> **Datatype:** String
 | `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.<br> **Datatype:** Boolean
 | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
@@ -141,7 +163,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 
 ### Parameters in the strategy
 
-The following parameters can be set in configuration file or strategy.
+The following parameters can be set in the configuration file or strategy.
 Values set in the configuration file always overwrite values set in the strategy.
 
 * `minimal_roi`
@@ -157,51 +179,67 @@ Values set in the configuration file always overwrite values set in the strategy
 * `order_time_in_force`
 * `unfilledtimeout`
 * `disable_dataframe_checks`
-* `use_sell_signal` (ask_strategy)
-* `sell_profit_only` (ask_strategy)
-* `sell_profit_offset` (ask_strategy)
-* `ignore_roi_if_buy_signal` (ask_strategy)
-* `ignore_buying_expired_candle_after` (ask_strategy)
+* `use_sell_signal`
+* `sell_profit_only`
+* `sell_profit_offset`
+* `ignore_roi_if_buy_signal`
+* `ignore_buying_expired_candle_after`
 
 ### Configuring amount per trade
 
-There are several methods to configure how much of the stake currency the bot will use to enter a trade. All methods respect the [available balance configuration](#available-balance) as explained below.
+There are several methods to configure how much of the stake currency the bot will use to enter a trade. All methods respect the [available balance configuration](#tradable-balance) as explained below.
 
 #### Minimum trade stake
 
-The minimum stake amount will depend by exchange and pair, and is usually listed in the exchange support pages.
+The minimum stake amount will depend on exchange and pair and is usually listed in the exchange support pages.
 Assuming the minimum tradable amount for XRP/USD is 20 XRP (given by the exchange), and the price is 0.6$.
 
-The minimum stake amount to buy this pair is therefore `20 * 0.6 ~= 12`.
+The minimum stake amount to buy this pair is, therefore, `20 * 0.6 ~= 12`.
 This exchange has also a limit on USD - where all orders must be > 10$ - which however does not apply in this case.
 
 To guarantee safe execution, freqtrade will not allow buying with a stake-amount of 10.1$, instead, it'll make sure that there's enough space to place a stoploss below the pair (+ an offset, defined by `amount_reserve_percent`, which defaults to 5%).
 
-With a reserve of 5%, the minimum stake amount would be ~12.6$ (`12 * (1 + 0.05)`). If we take in account a stoploss of 10% on top of that - we'd end up with a value of ~14$ (`12.6 / (1 - 0.1)`).
+With a reserve of 5%, the minimum stake amount would be ~12.6$ (`12 * (1 + 0.05)`). If we take into account a stoploss of 10% on top of that - we'd end up with a value of ~14$ (`12.6 / (1 - 0.1)`).
 
 To limit this calculation in case of large stoploss values, the calculated minimum stake-limit will never be more than 50% above the real limit.
 
 !!! Warning
     Since the limits on exchanges are usually stable and are not updated often, some pairs can show pretty high minimum limits, simply because the price increased a lot since the last limit adjustment by the exchange.
 
-#### Available balance
+#### Tradable balance
 
 By default, the bot assumes that the `complete amount - 1%` is at it's disposal, and when using [dynamic stake amount](#dynamic-stake-amount), it will split the complete balance into `max_open_trades` buckets per trade.
 Freqtrade will reserve 1% for eventual fees when entering a trade and will therefore not touch that by default.
 
 You can configure the "untouched" amount by using the `tradable_balance_ratio` setting.
 
-For example, if you have 10 ETH available in your wallet on the exchange and `tradable_balance_ratio=0.5` (which is 50%), then the bot will use a maximum amount of 5 ETH for trading and considers this as available balance. The rest of the wallet is untouched by the trades.
+For example, if you have 10 ETH available in your wallet on the exchange and `tradable_balance_ratio=0.5` (which is 50%), then the bot will use a maximum amount of 5 ETH for trading and considers this as an available balance. The rest of the wallet is untouched by the trades.
+
+!!! Danger
+    This setting should **not** be used when running multiple bots on the same account. Please look at [Available Capital to the bot](#assign-available-capital) instead.
 
 !!! Warning
-    The `tradable_balance_ratio` setting applies to the current balance (free balance + tied up in trades). Therefore, assuming the starting balance of 1000, a configuration with `tradable_balance_ratio=0.99` will not guarantee that 10 currency units will always remain available on the exchange. For example, the free amount may reduce to 5 units if the total balance is reduced to 500 (either by a losing streak, or by withdrawing balance).
+    The `tradable_balance_ratio` setting applies to the current balance (free balance + tied up in trades). Therefore, assuming the starting balance of 1000, a configuration with `tradable_balance_ratio=0.99` will not guarantee that 10 currency units will always remain available on the exchange. For example, the free amount may reduce to 5 units if the total balance is reduced to 500 (either by a losing streak or by withdrawing balance).
+
+#### Assign available Capital
+
+To fully utilize compounding profits when using multiple bots on the same exchange account, you'll want to limit each bot to a certain starting balance.
+This can be accomplished by setting `available_capital` to the desired starting balance.
+
+Assuming your account has 10.000 USDT and you want to run 2 different strategies on this exchange.
+You'd set `available_capital=5000` - granting each bot an initial capital of 5000 USDT.
+The bot will then split this starting balance equally into `max_open_trades` buckets.
+Profitable trades will result in increased stake-sizes for this bot - without affecting the stake-sizes of the other bot.
+
+!!! Warning "Incompatible with `tradable_balance_ratio`"
+    Setting this option will replace any configuration of `tradable_balance_ratio`.
 
 #### Amend last stake amount
 
 Assuming we have the tradable balance of 1000 USDT, `stake_amount=400`, and `max_open_trades=3`.
-The bot would open 2 trades, and will be unable to fill the last trading slot, since the requested 400 USDT are no longer available, since 800 USDT are already tied in other trades.
+The bot would open 2 trades and will be unable to fill the last trading slot, since the requested 400 USDT are no longer available since 800 USDT are already tied in other trades.
 
-To overcome this, the option `amend_last_stake_amount` can be set to `True`, which will enable the bot to reduce stake_amount to the available balance in order to fill the last trade slot.
+To overcome this, the option `amend_last_stake_amount` can be set to `True`, which will enable the bot to reduce stake_amount to the available balance to fill the last trade slot.
 
 In the example above this would mean:
 
@@ -229,7 +267,7 @@ For example, the bot will at most use (0.05 BTC x 3) = 0.15 BTC, assuming a conf
 
 #### Dynamic stake amount
 
-Alternatively, you can use a dynamic stake amount, which will use the available balance on the exchange, and divide that equally by the amount of allowed trades (`max_open_trades`).
+Alternatively, you can use a dynamic stake amount, which will use the available balance on the exchange, and divide that equally by the number of allowed trades (`max_open_trades`).
 
 To configure this, set `stake_amount="unlimited"`. We also recommend to set `tradable_balance_ratio=0.99` (99%) - to keep a minimum balance for eventual fees.
 
@@ -247,18 +285,18 @@ To allow the bot to trade all the available `stake_currency` in your account (mi
 ```
 
 !!! Tip "Compounding profits"
-    This configuration will allow increasing / decreasing stakes depending on the performance of the bot (lower stake if bot is loosing, higher stakes if the bot has a winning record, since higher balances are available), and will result in profit compounding.
+    This configuration will allow increasing/decreasing stakes depending on the performance of the bot (lower stake if the bot is losing, higher stakes if the bot has a winning record since higher balances are available), and will result in profit compounding.
 
 !!! Note "When using Dry-Run Mode"
-    When using `"stake_amount" : "unlimited",` in combination with Dry-Run, Backtesting or Hyperopt, the balance will be simulated starting with a stake of `dry_run_wallet` which will evolve over time.  
-    It is therefore important to set `dry_run_wallet` to a sensible value (like 0.05 or 0.01 for BTC and 1000 or 100 for USDT, for example), otherwise it may simulate trades with 100 BTC (or more) or 0.05 USDT (or less) at once - which may not correspond to your real available balance or is less than the exchange minimal limit for the order amount for the stake currency.
+    When using `"stake_amount" : "unlimited",` in combination with Dry-Run, Backtesting or Hyperopt, the balance will be simulated starting with a stake of `dry_run_wallet` which will evolve.
+    It is therefore important to set `dry_run_wallet` to a sensible value (like 0.05 or 0.01 for BTC and 1000 or 100 for USDT, for example), otherwise, it may simulate trades with 100 BTC (or more) or 0.05 USDT (or less) at once - which may not correspond to your real available balance or is less than the exchange minimal limit for the order amount for the stake currency.
 
 --8<-- "includes/pricing.md"
 
 ### Understand minimal_roi
 
 The `minimal_roi` configuration parameter is a JSON object where the key is a duration
-in minutes and the value is the minimum ROI as ratio.
+in minutes and the value is the minimum ROI as a ratio.
 See the example below:
 
 ```json
@@ -273,7 +311,7 @@ See the example below:
 Most of the strategy files already include the optimal `minimal_roi` value.
 This parameter can be set in either Strategy or Configuration file. If you use it in the configuration file, it will override the
 `minimal_roi` value from the strategy file.
-If it is not set in either Strategy or Configuration, a default of 1000% `{"0": 10}` is used, and minimal roi is disabled unless your trade generates 1000% profit.
+If it is not set in either Strategy or Configuration, a default of 1000% `{"0": 10}` is used, and minimal ROI is disabled unless your trade generates 1000% profit.
 
 !!! Note "Special case to forcesell after a specific time"
     A special case presents using `"<N>": -1` as ROI. This forces the bot to sell a trade after N Minutes, no matter if it's positive or negative, so represents a time-limited force-sell.
@@ -292,16 +330,16 @@ See [the telegram documentation](telegram-usage.md) for details on usage.
 
 When working with larger timeframes (for example 1h or more) and using a low `max_open_trades` value, the last candle can be processed as soon as a trade slot becomes available. When processing the last candle, this can lead to a situation where it may not be desirable to use the buy signal on that candle. For example, when using a condition in your strategy where you use a cross-over, that point may have passed too long ago for you to start a trade on it.
 
-In these situations, you can enable the functionality to ignore candles that are beyond a specified period by setting `ask_strategy.ignore_buying_expired_candle_after` to a positive number, indicating the number of seconds after which the buy signal becomes expired.
+In these situations, you can enable the functionality to ignore candles that are beyond a specified period by setting `ignore_buying_expired_candle_after` to a positive number, indicating the number of seconds after which the buy signal becomes expired.
 
 For example, if your strategy is using a 1h timeframe, and you only want to buy within the first 5 minutes when a new candle comes in, you can add the following configuration to your strategy:
 
 ``` json
-  "ask_strategy":{
+  {
+    //...
     "ignore_buying_expired_candle_after": 300,
-    "price_side": "bid",
     // ...
-  },
+  }
 ```
 
 !!! Note
@@ -319,7 +357,7 @@ the buy order is fulfilled.
 `order_types` set in the configuration file overwrites values set in the strategy as a whole, so you need to configure the whole `order_types` dictionary in one place.
 
 If this is configured, the following 4 values (`buy`, `sell`, `stoploss` and
-`stoploss_on_exchange`) need to be present, otherwise the bot will fail to start.
+`stoploss_on_exchange`) need to be present, otherwise, the bot will fail to start.
 
 For information on (`emergencysell`,`forcesell`, `forcebuy`, `stoploss_on_exchange`,`stoploss_on_exchange_interval`,`stoploss_on_exchange_limit_ratio`) please see stop loss documentation [stop loss on exchange](stoploss.md)
 
@@ -370,7 +408,7 @@ Configuration:
     If `stoploss_on_exchange` is enabled and the stoploss is cancelled manually on the exchange, then the bot will create a new stoploss order.
 
 !!! Warning "Warning: stoploss_on_exchange failures"
-    If stoploss on exchange creation fails for some reason, then an "emergency sell" is initiated. By default, this will sell the asset using a market order. The order-type for the emergency-sell can be changed by setting the `emergencysell` value in the `order_types` dictionary - however this is not advised.
+    If stoploss on exchange creation fails for some reason, then an "emergency sell" is initiated. By default, this will sell the asset using a market order. The order-type for the emergency-sell can be changed by setting the `emergencysell` value in the `order_types` dictionary - however, this is not advised.
 
 ### Understand order_time_in_force
 
@@ -380,12 +418,12 @@ is executed on the exchange. Three commonly used time in force are:
 **GTC (Good Till Canceled):**
 
 This is most of the time the default time in force. It means the order will remain
-on exchange till it is canceled by user. It can be fully or partially fulfilled.
+on exchange till it is cancelled by the user. It can be fully or partially fulfilled.
 If partially fulfilled, the remaining will stay on the exchange till cancelled.
 
 **FOK (Fill Or Kill):**
 
-It means if the order is not executed immediately AND fully then it is canceled by the exchange.
+It means if the order is not executed immediately AND fully then it is cancelled by the exchange.
 
 **IOC (Immediate Or Canceled):**
 
@@ -406,7 +444,7 @@ The possible values are: `gtc` (default), `fok` or `ioc`.
 ```
 
 !!! Warning
-    This is an ongoing work. For now it is supported only for binance.
+    This is ongoing work. For now, it is supported only for binance.
     Please don't change the default value unless you know what you are doing and have researched the impact of using different values.
 
 ### Exchange configuration
@@ -415,7 +453,7 @@ Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports
 exchange markets and trading APIs. The complete up-to-date list can be found in the
 [CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python).
  However, the bot was tested by the development team with only Bittrex, Binance and Kraken,
- so the these are the only officially supported exchanges:
+ so these are the only officially supported exchanges:
 
 - [Bittrex](https://bittrex.com/): "bittrex"
 - [Binance](https://www.binance.com/): "binance"
@@ -441,11 +479,11 @@ A exchange configuration for "binance" would look as follows:
     },
 ```
 
-This configuration enables binance, as well as rate limiting to avoid bans from the exchange.
+This configuration enables binance, as well as rate-limiting to avoid bans from the exchange.
 `"rateLimit": 200` defines a wait-event of 0.2s between each call. This can also be completely disabled by setting `"enableRateLimit"` to false.
 
 !!! Note
-    Optimal settings for rate limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings.
+    Optimal settings for rate-limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings.
     We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step.
 
 ### What values can be used for fiat_display_currency?
@@ -459,7 +497,7 @@ The valid values are:
 "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD"
 ```
 
-In addition to fiat currencies, a range of cryto currencies are supported.
+In addition to fiat currencies, a range of crypto currencies is supported.
 
 The valid values are:
 
@@ -470,7 +508,7 @@ The valid values are:
 ## Using Dry-run mode
 
 We recommend starting the bot in the Dry-run mode to see how your bot will
-behave and what is the performance of your strategy. In the Dry-run mode the
+behave and what is the performance of your strategy. In the Dry-run mode, the
 bot does not engage your money. It only runs a live simulation without
 creating trades on the exchange.
 
@@ -496,7 +534,7 @@ creating trades on the exchange.
 Once you will be happy with your bot performance running in the Dry-run mode, you can switch it to production mode.
 
 !!! Note
-    A simulated wallet is available during dry-run mode, and will assume a starting capital of `dry_run_wallet` (defaults to 1000).
+    A simulated wallet is available during dry-run mode and will assume a starting capital of `dry_run_wallet` (defaults to 1000).
 
 ### Considerations for dry-run
 
@@ -504,20 +542,21 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo
 * Wallets (`/balance`) are simulated based on `dry_run_wallet`.
 * Orders are simulated, and will not be posted to the exchange.
 * Market orders fill based on orderbook volume the moment the order is placed.
-* Limit orders fill once price reaches the defined level - or time out based on `unfilledtimeout` settings.
+* Limit orders fill once the price reaches the defined level - or time out based on `unfilledtimeout` settings.
 * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled.
 * Open orders (not trades, which are stored in the database) are reset on bot restart.
 
 ## Switch to production mode
 
-In production mode, the bot will engage your money. Be careful, since a wrong
-strategy can lose all your money. Be aware of what you are doing when
-you run it in production mode.
+In production mode, the bot will engage your money. Be careful, since a wrong strategy can lose all your money.
+Be aware of what you are doing when you run it in production mode.
+
+When switching to Production mode, please make sure to use a different / fresh database to avoid dry-run trades messing with your exchange money and eventually tainting your statistics.
 
 ### Setup your exchange account
 
 You will need to create API Keys (usually you get `key` and `secret`, some exchanges require an additional `password`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the `freqtrade new-config` command.
-API Keys are usually only required for live trading (trading for real money, bot running in "production mode", executing real orders on the exchange) and are not required for the bot running in dry-run (trade simulation) mode. When you setup the bot in dry-run mode, you may fill these fields with empty values.
+API Keys are usually only required for live trading (trading for real money, bot running in "production mode", executing real orders on the exchange) and are not required for the bot running in dry-run (trade simulation) mode. When you set up the bot in dry-run mode, you may fill these fields with empty values.
 
 ### To switch your bot in production mode
 
@@ -529,7 +568,7 @@ API Keys are usually only required for live trading (trading for real money, bot
 "dry_run": false,
 ```
 
-**Insert your Exchange API key (change them by fake api keys):**
+**Insert your Exchange API key (change them by fake API keys):**
 
 ```json
 {
@@ -547,7 +586,7 @@ API Keys are usually only required for live trading (trading for real money, bot
 You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange.
 
 !!! Hint "Keep your secrets secret"
-    To keep your secrets secret, we recommend to use a 2nd configuration for your API keys.
+    To keep your secrets secret, we recommend using a 2nd configuration for your API keys.
     Simply use the above snippet in a new configuration file (e.g. `config-private.json`) and keep your settings in this file.
     You can then start the bot with `freqtrade trade --config user_data/config.json --config user_data/config-private.json <...>` to have your keys loaded.
 
@@ -557,7 +596,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d
 
 To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration.
 
-An example for this can be found in `config_full.json.example`
+An example for this can be found in `config_examples/config_full.example.json`
 
 ``` json
 "ccxt_async_config": {

docs/data-download.md

@@ -271,7 +271,7 @@ mkdir -p user_data/data/binance
 cp tests/testdata/pairs.json user_data/data/binance
 ```
 
-If you your configuration directory `user_data` was made by docker, you may get the following error:
+If your configuration directory `user_data` was made by docker, you may get the following error:
 
 ```
 cp: cannot create regular file 'user_data/data/binance/pairs.json': Permission denied

docs/deprecated.md

@@ -33,3 +33,8 @@ The old section of configuration parameters (`"pairlist"`) has been deprecated i
 ### deprecation of bidVolume and askVolume from volume-pairlist
 
 Since only quoteVolume can be compared between assets, the other options (bidVolume, askVolume) have been deprecated in 2020.4, and have been removed in 2020.9.
+
+### Using order book steps for sell price
+
+Using `order_book_min` and `order_book_max` used to allow stepping the orderbook and trying to find the next ROI slot - trying to place sell-orders early.
+As this does however increase risk and provides no benefit, it's been removed for maintainability purposes in 2021.7.

docs/developer.md

@@ -2,7 +2,7 @@
 
 This page is intended for developers of Freqtrade, people who want to contribute to the Freqtrade codebase or documentation, or people who want to understand the source code of the application they're running.
 
-All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/p7nuUNVfP7) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) where you can ask questions.
+All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/p7nuUNVfP7) where you can ask questions.
 
 ## Documentation
 
@@ -240,11 +240,18 @@ The `IProtection` parent class provides a helper method for this in `calculate_l
 !!! Note
     This section is a Work in Progress and is not a complete guide on how to test a new exchange with Freqtrade.
 
+!!! Note
+    Make sure to use an up-to-date version of CCXT before running any of the below tests.
+    You can get the latest version of ccxt by running `pip install -U ccxt` with activated virtual environment.
+    Native docker is not supported for these tests, however the available dev-container will support all required actions and eventually necessary changes.
+
 Most exchanges supported by CCXT should work out of the box.
 
 To quickly test the public endpoints of an exchange, add a configuration for your exchange to `test_ccxt_compat.py` and run these tests with `pytest --longrun tests/exchange/test_ccxt_compat.py`.
 Completing these tests successfully a good basis point (it's a requirement, actually), however these won't guarantee correct exchange functioning, as this only tests public endpoints, but no private endpoint (like generate order or similar).
 
+Also try to use `freqtrade download-data` for an extended timerange and verify that the data downloaded correctly (no holes, the specified timerange was actually downloaded).
+
 ### Stoploss On Exchange
 
 Check if the new exchange supports Stoploss on Exchange orders through their API.

docs/docker_quickstart.md

@@ -24,82 +24,21 @@ Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.co
 
 Create a new directory and place the [docker-compose file](https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml) in this directory.
 
-=== "PC/MAC/Linux"
-    ``` bash
-    mkdir ft_userdata
-    cd ft_userdata/
-    # Download the docker-compose file from the repository
-    curl https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml -o docker-compose.yml
+``` bash
+mkdir ft_userdata
+cd ft_userdata/
+# Download the docker-compose file from the repository
+curl https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml -o docker-compose.yml
 
-    # Pull the freqtrade image
-    docker-compose pull
+# Pull the freqtrade image
+docker-compose pull
 
-    # Create user directory structure
-    docker-compose run --rm freqtrade create-userdir --userdir user_data
+# Create user directory structure
+docker-compose run --rm freqtrade create-userdir --userdir user_data
 
-    # Create configuration - Requires answering interactive questions
-    docker-compose run --rm freqtrade new-config --config user_data/config.json
-    ```
-
-=== "RaspberryPi"
-    ``` bash
-    mkdir ft_userdata
-    cd ft_userdata/
-    # Download the docker-compose file from the repository
-    curl https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml -o docker-compose.yml
-
-    # Edit the compose file to use an image named `*_pi` (stable_pi or develop_pi)
-
-    # Pull the freqtrade image
-    docker-compose pull
-
-    # Create user directory structure
-    docker-compose run --rm freqtrade create-userdir --userdir user_data
-
-    # Create configuration - Requires answering interactive questions
-    docker-compose run --rm freqtrade new-config --config user_data/config.json
-    ```
-
-    !!! Note "Change your docker Image"
-        You have to change the docker image in the docker-compose file for your Raspberry build to work properly.
-        ``` yml
-        image: freqtradeorg/freqtrade:stable_pi
-        # image: freqtradeorg/freqtrade:develop_pi
-        ```
-
-=== "ARM 64 Systenms (Mac M1, Raspberry Pi 4, Jetson Nano)"
-    In case of a Mac M1, make sure that your docker installation is running in native mode
-    Arm64 images are not yet provided via Docker Hub and need to be build locally first.
-    Depending on the device, this may take a few minutes (Apple M1) or multiple hours (Raspberry Pi)
-
-    ``` bash
-    # Clone Freqtrade repository
-    git clone  https://github.com/freqtrade/freqtrade.git
-    cd freqtrade
-    # Optionally switch to the stable version
-    git checkout stable   
-
-    # Modify your docker-compose file to enable building and change the image name 
-    # (see the Note Box below for necessary changes)
-
-    # Build image
-    docker-compose build
-    
-    # Create user directory structure
-    docker-compose run --rm freqtrade create-userdir --userdir user_data
-
-    # Create configuration - Requires answering interactive questions
-    docker-compose run --rm freqtrade new-config --config user_data/config.json
-    ```
-
-    !!! Note "Change your docker Image"
-        You have to change the docker image in the docker-compose file for your arm64 build to work properly.
-        ``` yml
-        image: freqtradeorg/freqtrade:custom_arm64
-        build:
-          context: .
-          dockerfile: "Dockerfile"
-        ```
+# Create configuration - Requires answering interactive questions
+docker-compose run --rm freqtrade new-config --config user_data/config.json
+```
 
 The above snippet creates a new directory called `ft_userdata`, downloads the latest compose file and pulls the freqtrade image.
 The last 2 steps in the snippet create the directory with `user_data`, as well as (interactively) the default configuration based on your selections.
@@ -117,7 +56,7 @@ The last 2 steps in the snippet create the directory with `user_data`, as well a
 
 The `SampleStrategy` is run by default.
 
-!!! Warning "`SampleStrategy` is just a demo!"
+!!! Danger "`SampleStrategy` is just a demo!"
     The `SampleStrategy` is there for your reference and give you ideas for your own strategy.
     Please always backtest your strategy and use dry-run for some time before risking real money!
     You will find more information about Strategy development in the [Strategy documentation](strategy-customization.md).
@@ -167,6 +106,10 @@ Advanced users may edit the docker-compose file further to include all possible
 
 All freqtrade arguments will be available by running `docker-compose run --rm freqtrade <command> <optional arguments>`.
 
+!!! Warning "`docker-compose` for trade commands"
+    Trade commands (`freqtrade trade <...>`) should not be ran via `docker-compose run` - but should use `docker-compose up -d` instead.
+    This makes sure that the container is properly started (including port forwardings) and will make sure that the container will restart after a system reboot.
+
 !!! Note "`docker-compose run --rm`"
     Including `--rm` will remove the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command).
 

docs/exchanges.md

@@ -77,8 +77,9 @@ You can get a list of restricted markets by using the following snippet:
 ``` python
 import ccxt
 ct = ccxt.bittrex()
-_ = ct.load_markets()
-res = [ f"{x['MarketCurrency']}/{x['BaseCurrency']}" for x in ct.publicGetMarkets()['result'] if x['IsRestricted']]
+lm = ct.load_markets()
+
+res = [p for p, x in lm.items() if 'US' in x['info']['prohibitedIn']]
 print(res)
 ```
 
@@ -104,7 +105,7 @@ To use subaccounts with FTX, you need to edit the configuration and add the foll
 
 ## Kucoin
 
-Kucoin requries a passphrase for each api key, you will therefore need to add this key into the configuration so your exchange section looks as follows:
+Kucoin requires a passphrase for each api key, you will therefore need to add this key into the configuration so your exchange section looks as follows:
 
 ```json
 "exchange": {

docs/faq.md

@@ -172,7 +172,7 @@ freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossD
 
 ### Why does it take a long time to run hyperopt?
 
-* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/p7nuUNVfP7). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you.
+* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [discord community](https://discord.gg/p7nuUNVfP7). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you.
 
 * If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers:
 

docs/hyperopt.md

@@ -48,10 +48,10 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
                           [--hyperopt-path PATH] [--eps] [--dmmp]
                           [--enable-protections]
                           [--dry-run-wallet DRY_RUN_WALLET] [-e INT]
-                          [--spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]]
+                          [--spaces {all,buy,sell,roi,stoploss,trailing,protection,default} [{all,buy,sell,roi,stoploss,trailing,protection,default} ...]]
                           [--print-all] [--no-color] [--print-json] [-j JOBS]
                           [--random-state INT] [--min-trades INT]
-                          [--hyperopt-loss NAME]
+                          [--hyperopt-loss NAME] [--disable-param-export]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -92,7 +92,7 @@ optional arguments:
                         Starting balance, used for backtesting / hyperopt and
                         dry-runs.
   -e INT, --epochs INT  Specify number of epochs (default: 100).
-  --spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]
+  --spaces {all,buy,sell,roi,stoploss,trailing,protection,default} [{all,buy,sell,roi,stoploss,trailing,protection,default} ...]
                         Specify which parameters to hyperopt. Space-separated
                         list.
   --print-all           Print all results, not only the best ones.
@@ -118,6 +118,8 @@ optional arguments:
                         ShortTradeDurHyperOptLoss, OnlyProfitHyperOptLoss,
                         SharpeHyperOptLoss, SharpeHyperOptLossDaily,
                         SortinoHyperOptLoss, SortinoHyperOptLossDaily
+  --disable-param-export
+                        Disable automatic hyperopt parameter export.
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
@@ -251,7 +253,7 @@ We continue to define hyperoptable parameters:
 class MyAwesomeStrategy(IStrategy):
     buy_adx = DecimalParameter(20, 40, decimals=1, default=30.1, space="buy")
     buy_rsi = IntParameter(20, 40, default=30, space="buy")
-    buy_adx_enabled = CategoricalParameter([True, False], default=True, space="buy")
+    buy_adx_enabled = BooleanParameter(default=True, space="buy")
     buy_rsi_enabled = CategoricalParameter([True, False], default=False, space="buy")
     buy_trigger = CategoricalParameter(["bb_lower", "macd_cross_signal"], default="bb_lower", space="buy")
 ```
@@ -314,6 +316,7 @@ There are four parameter types each suited for different purposes.
 * `DecimalParameter` - defines a floating point parameter with a limited number of decimals (default 3). Should be preferred instead of `RealParameter` in most cases.
 * `RealParameter` - defines a floating point parameter with upper and lower boundaries and no precision limit. Rarely used as it creates a space with a near infinite number of possibilities.
 * `CategoricalParameter` - defines a parameter with a predetermined number of choices.
+* `BooleanParameter` - Shorthand for `CategoricalParameter([True, False])` - great for "enable" parameters.
 
 !!! Tip "Disabling parameter optimization"
     Each parameter takes two boolean parameters:
@@ -324,7 +327,7 @@ There are four parameter types each suited for different purposes.
 !!! Warning
     Hyperoptable parameters cannot be used in `populate_indicators` - as hyperopt does not recalculate indicators for each epoch, so the starting value would be used in this case.
 
-### Optimizing an indicator parameter
+## Optimizing an indicator parameter
 
 Assuming you have a simple strategy in mind - a EMA cross strategy (2 Moving averages crossing) - and you'd like to find the ideal parameters for this strategy.
 
@@ -334,8 +337,8 @@ from functools import reduce
 
 import talib.abstract as ta
 
-from freqtrade.strategy import IStrategy
-from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter
+from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter, 
+                                IStrategy, IntParameter)
 import freqtrade.vendor.qtpylib.indicators as qtpylib
 
 class MyAwesomeStrategy(IStrategy):
@@ -403,11 +406,106 @@ While this strategy is most likely too simple to provide consistent profit, it s
 !!! Note
     `self.buy_ema_short.range` will act differently between hyperopt and other modes. For hyperopt, the above example may generate 48 new columns, however for all other modes (backtesting, dry/live), it will only generate the column for the selected value. You should therefore avoid using the resulting column with explicit values (values other than `self.buy_ema_short.value`).
 
+!!! Note
+    `range` property may also be used with `DecimalParameter` and `CategoricalParameter`. `RealParameter` does not provide this property due to infinite search space.
+
 ??? Hint "Performance tip"
     By doing the calculation of all possible indicators in `populate_indicators()`, the calculation of the indicator happens only once for every parameter.  
     While this may slow down the hyperopt startup speed, the overall performance will increase as the Hyperopt execution itself may pick the same value for multiple epochs (changing other values).
     You should however try to use space ranges as small as possible. Every new column will require more memory, and every possibility hyperopt can try will increase the search space.
 
+## Optimizing protections
+
+Freqtrade can also optimize protections. How you optimize protections is up to you, and the following should be considered as example only.
+
+The strategy will simply need to define the "protections" entry as property returning a list of protection configurations.
+
+``` python
+from pandas import DataFrame
+from functools import reduce
+
+import talib.abstract as ta
+
+from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter, 
+                                IStrategy, IntParameter)
+import freqtrade.vendor.qtpylib.indicators as qtpylib
+
+class MyAwesomeStrategy(IStrategy):
+    stoploss = -0.05
+    timeframe = '15m'
+    # Define the parameter spaces
+    cooldown_lookback = IntParameter(2, 48, default=5, space="protection", optimize=True)
+    stop_duration = IntParameter(12, 200, default=5, space="protection", optimize=True)
+    use_stop_protection = BooleanParameter(default=True, space="protection", optimize=True)
+
+
+    @property
+    def protections(self):
+        prot = []
+
+        prot.append({
+            "method": "CooldownPeriod",
+            "stop_duration_candles": self.cooldown_lookback.value
+        })
+        if self.use_stop_protection.value:
+            prot.append({
+                "method": "StoplossGuard",
+                "lookback_period_candles": 24 * 3,
+                "trade_limit": 4,
+                "stop_duration_candles": self.stop_duration.value,
+                "only_per_pair": False
+            })
+
+        return protection
+
+    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        # ...
+        
+```
+
+You can then run hyperopt as follows:
+`freqtrade hyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy MyAwesomeStrategy --spaces protection`
+
+!!! Note
+    The protection space is not part of the default space, and is only available with the Parameters Hyperopt interface, not with the legacy hyperopt interface (which required separate hyperopt files).
+    Freqtrade will also automatically change the "--enable-protections" flag if the protection space is selected.
+
+!!! Warning
+    If protections are defined as property, entries from the configuration will be ignored.
+    It is therefore recommended to not define protections in the configuration.
+
+### Migrating from previous property setups
+
+A migration from a previous setup is pretty simple, and can be accomplished by converting the protections entry to a property.
+In simple terms, the following configuration will be converted to the below.
+
+``` python
+class MyAwesomeStrategy(IStrategy):
+    protections = [
+        {
+            "method": "CooldownPeriod",
+            "stop_duration_candles": 4
+        }
+    ]
+```
+
+Result
+
+``` python
+class MyAwesomeStrategy(IStrategy):
+    
+    @property
+    def protections(self):
+        return [
+            {
+                "method": "CooldownPeriod",
+                "stop_duration_candles": 4
+            }
+        ]
+```
+
+You will then obviously also change potential interesting entries to parameters to allow hyper-optimization.
+
 ## Loss-functions
 
 Each hyperparameter tuning requires a target. This is usually defined as a loss function (sometimes also called objective function), which should decrease for more desirable results, and increase for bad results.
@@ -478,7 +576,8 @@ Legal values are:
 * `roi`: just optimize the minimal profit table for your strategy
 * `stoploss`: search for the best stoploss value
 * `trailing`: search for the best trailing stop values
-* `default`: `all` except `trailing`
+* `protection`: search for the best protection parameters (read the [protections section](#optimizing-protections) on how to properly define these)
+* `default`: `all` except `trailing` and `protection`
 * space-separated list of any of the above values for example `--spaces roi stoploss`
 
 The default Hyperopt Search Space, used when no `--space` command line option is specified, does not include the `trailing` hyperspace. We recommend you to run optimization for the `trailing` hyperspace separately, when the best parameters for other hyperspaces were found, validated and pasted into your custom strategy.
@@ -509,7 +608,13 @@ You should understand this result like:
 * You should not use ADX because `'buy_adx_enabled': False`.
 * You should **consider** using the RSI indicator (`'buy_rsi_enabled': True`) and the best value is `29.0` (`'buy_rsi': 29.0`)
 
-Your strategy class can immediately take advantage of these results. Simply copy hyperopt results block and paste them at class level, replacing old parameters (if any). New parameters will automatically be loaded next time strategy is executed.
+### Automatic parameter application to the strategy
+
+When using Hyperoptable parameters, the result of your hyperopt-run will be written to a json file next to your strategy (so for `MyAwesomeStrategy.py`, the file would be `MyAwesomeStrategy.json`).  
+This file is also updated when using the `hyperopt-show` sub-command, unless `--disable-param-export` is provided to either of the 2 commands.
+
+
+Your strategy class can also contain these results explicitly. Simply copy hyperopt results block and paste them at class level, replacing old parameters (if any). New parameters will automatically be loaded next time strategy is executed.
 
 Transferring your whole hyperopt result to your strategy would then look like:
 
@@ -525,6 +630,10 @@ class MyAwesomeStrategy(IStrategy):
     }
 ```
 
+!!! Note
+    Values in the configuration file will overwrite Parameter-file level parameters - and both will overwrite parameters within the strategy.
+    The prevalence is therefore: config > parameter file > strategy
+
 ### Understand Hyperopt ROI results
 
 If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'default' or 'roi'), your result will look as follows and include a ROI table:

docs/includes/pairlists.md

@@ -23,6 +23,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
 * [`StaticPairList`](#static-pair-list) (default, if not configured differently)
 * [`VolumePairList`](#volume-pair-list)
 * [`AgeFilter`](#agefilter)
+* [`OffsetFilter`](#offsetfilter)
 * [`PerformanceFilter`](#performancefilter)
 * [`PrecisionFilter`](#precisionfilter)
 * [`PriceFilter`](#pricefilter)
@@ -57,23 +58,67 @@ This option must be configured along with `exchange.skip_pair_validation` in the
 
 When used in the chain of Pairlist Handlers in a non-leading position (after StaticPairList and other Pairlist Filters), `VolumePairList` considers outputs of previous Pairlist Handlers, adding its sorting/selection of the pairs by the trading volume.
 
-When used on the leading position of the chain of Pairlist Handlers, it does not consider `pair_whitelist` configuration setting, but selects the top assets from all available markets (with matching stake-currency) on the exchange.
+When used in the leading position of the chain of Pairlist Handlers, the `pair_whitelist` configuration setting is ignored. Instead, `VolumePairList` selects the top assets from all available markets with matching stake-currency on the exchange.
 
 The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes).
 The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists.
 Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data.
 
-`VolumePairList` is based on the ticker data from exchange, as reported by the ccxt library:
+`VolumePairList` is per default based on the ticker data from exchange, as reported by the ccxt library:
 
 * The `quoteVolume` is the amount of quote (stake) currency traded (bought or sold) in last 24 hours.
 
 ```json
-"pairlists": [{
+"pairlists": [
+    {
         "method": "VolumePairList",
         "number_assets": 20,
         "sort_key": "quoteVolume",
+        "min_value": 0,
         "refresh_period": 1800
-}],
+    }
+],
+```
+
+You can define a minimum volume with `min_value` - which will filter out pairs with a volume lower than the specified value in the specified timerange.
+
+`VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a  more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
+
+For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days:
+
+```json
+"pairlists": [
+    {
+        "method": "VolumePairList",
+        "number_assets": 20,
+        "sort_key": "quoteVolume",
+        "min_value": 0,
+        "refresh_period": 86400,
+        "lookback_days": 7
+    }
+],
+```
+
+!!! Warning "Range look back and refresh period"
+    When used in conjunction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API.
+
+!!! Warning "Performance implications when using lookback range"
+    If used in first position in combination with lookback, the computation of the range based volume can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation.
+
+More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles:
+
+```json
+"pairlists": [
+    {
+        "method": "VolumePairList",
+        "number_assets": 20,
+        "sort_key": "quoteVolume",
+        "min_value": 0,
+        "refresh_period": 3600,
+        "lookback_timeframe": "1h",
+        "lookback_period": 72
+    }
+],
 ```
 
 !!! Note
@@ -81,13 +126,39 @@ Filtering instances (not the first position in the list) will not apply any cach
 
 #### AgeFilter
 
-Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`).
+Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity).
 
 When pairs are first listed on an exchange they can suffer huge price drops and volatility
 in the first few days while the pair goes through its price-discovery period. Bots can often
 be caught out buying before the pair has finished dropping in price.
 
-This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days.
+This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days and listed before `max_days_listed`.
+
+#### OffsetFilter
+
+Offsets an incoming pairlist by a given `offset` value.
+
+As an example it can be used in conjunction with `VolumeFilter` to remove the top X volume pairs. Or to split
+a larger pairlist on two bot instances.
+
+Example to remove the first 10 pairs from the pairlist:
+
+```json
+"pairlists": [
+    {
+        "method": "OffsetFilter",
+        "offset": 10
+    }
+],
+```
+
+!!! Warning
+    When `OffsetFilter` is used to split a larger pairlist among multiple bots in combination with `VolumeFilter` 
+    it can not be guaranteed that pairs won't overlap due to slightly different refresh intervals for the
+    `VolumeFilter`.
+
+!!! Note
+    An offset larger then the total length of the incoming pairlist will result in an empty pairlist.
 
 #### PerformanceFilter
 
@@ -155,10 +226,10 @@ If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027, the ratio
 
 #### RangeStabilityFilter
 
-Removes pairs where the difference between lowest low and highest high over `lookback_days` days is below `min_rate_of_change`. Since this is a filter that requires additional data, the results are cached for `refresh_period`.
+Removes pairs where the difference between lowest low and highest high over `lookback_days` days is below `min_rate_of_change` or above `max_rate_of_change`. Since this is a filter that requires additional data, the results are cached for `refresh_period`.
 
 In the below example:
-If the trading range over the last 10 days is <1%, remove the pair from the whitelist.
+If the trading range over the last 10 days is <1% or >99%, remove the pair from the whitelist.
 
 ```json
 "pairlists": [
@@ -166,6 +237,7 @@ If the trading range over the last 10 days is <1%, remove the pair from the whit
         "method": "RangeStabilityFilter",
         "lookback_days": 10,
         "min_rate_of_change": 0.01,
+        "max_rate_of_change": 0.99,
         "refresh_period": 1440
     }
 ]
@@ -173,6 +245,7 @@ If the trading range over the last 10 days is <1%, remove the pair from the whit
 
 !!! Tip
     This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit.
+    Additionally, it can also be used to automatically remove pairs with extreme high/low variance over a given amount of time.
 
 #### VolatilityFilter
 

docs/includes/pricing.md

@@ -47,7 +47,7 @@ Also, prices at the "ask" side of the spread are higher than prices at the "bid"
 
 #### Buy price with Orderbook enabled
 
-When buying with the orderbook enabled (`bid_strategy.use_order_book=True`), Freqtrade fetches the `bid_strategy.order_book_top` entries from the orderbook and then uses the entry specified as `bid_strategy.order_book_top` on the configured side (`bid_strategy.price_side`) of the orderbook. 1 specifies the topmost entry in the orderbook, while 2 would use the 2nd entry in the orderbook, and so on.
+When buying with the orderbook enabled (`bid_strategy.use_order_book=True`), Freqtrade fetches the `bid_strategy.order_book_top` entries from the orderbook and uses the entry specified as `bid_strategy.order_book_top` on the configured side (`bid_strategy.price_side`) of the orderbook. 1 specifies the topmost entry in the orderbook, while 2 would use the 2nd entry in the orderbook, and so on.
 
 #### Buy price without Orderbook enabled
 
@@ -82,22 +82,9 @@ In line with that, if `ask_strategy.price_side` is set to `"bid"`, then the bot
 
 #### Sell price with Orderbook enabled
 
-When selling with the orderbook enabled (`ask_strategy.use_order_book=True`), Freqtrade fetches the `ask_strategy.order_book_max` entries in the orderbook. Then each of the orderbook steps between `ask_strategy.order_book_min` and `ask_strategy.order_book_max` on the configured orderbook side are validated for a profitable sell-possibility based on the strategy configuration (`minimal_roi` conditions) and the sell order is placed at the first profitable spot.
+When selling with the orderbook enabled (`ask_strategy.use_order_book=True`), Freqtrade fetches the `ask_strategy.order_book_top` entries in the orderbook and uses the entry specified as `ask_strategy.order_book_top` from the configured side (`ask_strategy.price_side`) as selling price.
 
-!!! Note
-    Using `order_book_max` higher than `order_book_min` only makes sense when ask_strategy.price_side is set to `"ask"`.
-
-The idea here is to place the sell order early, to be ahead in the queue.
-
-A fixed slot (mirroring `bid_strategy.order_book_top`) can be defined by setting `ask_strategy.order_book_min` and `ask_strategy.order_book_max` to the same number.
-
-!!! Warning "Order_book_max > 1 - increased risks for stoplosses!"
-    Using `ask_strategy.order_book_max` higher than 1 will increase the risk the stoploss on exchange is cancelled too early, since an eventual [stoploss on exchange](#understand-order_types) will be cancelled as soon as the order is placed.
-    Also, the sell order will remain on the exchange for `unfilledtimeout.sell` (or until it's filled) - which can lead to missed stoplosses (with or without using stoploss on exchange).
-
-!!! Warning "Order_book_max > 1 in dry-run"
-    Using `ask_strategy.order_book_max` higher than 1 will result in improper dry-run results (significantly better than real orders executed on exchange), since dry-run assumes orders to be filled almost instantly.
-    It is therefore advised to not use this setting for dry-runs.
+1 specifies the topmost entry in the orderbook, while 2 would use the 2nd entry in the orderbook, and so on.
 
 #### Sell price without Orderbook enabled
 

docs/includes/protections.md

@@ -1,7 +1,7 @@
 ## Protections
 
 !!! Warning "Beta feature"
-    This feature is still in it's testing phase. Should you notice something you think is wrong please let us know via Discord, Slack or via Github Issue.
+    This feature is still in it's testing phase. Should you notice something you think is wrong please let us know via Discord or via Github Issue.
 
 Protections will protect your strategy from unexpected events and market conditions by temporarily stop trading for either one pair, or for all pairs.
 All protection end times are rounded up to the next candle to avoid sudden, unexpected intra-candle buys.
@@ -15,6 +15,10 @@ All protection end times are rounded up to the next candle to avoid sudden, unex
 !!! Note "Backtesting"
     Protections are supported by backtesting and hyperopt, but must be explicitly enabled by using the `--enable-protections` flag.
 
+!!! Warning "Setting protections from the configuration"
+    Setting protections from the configuration via `"protections": [],` key should be considered deprecated and will be removed in a future version.
+    It is also no longer guaranteed that your protections apply to the strategy in cases where the strategy defines [protections as property](hyperopt.md#optimizing-protections).
+
 ### Available Protections
 
 * [`StoplossGuard`](#stoploss-guard) Stop trading if a certain amount of stoploss occurred within a certain time window.
@@ -47,15 +51,17 @@ This applies across all pairs, unless `only_per_pair` is set to true, which will
 The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles.
 
 ``` python
-protections = [
-    {
-        "method": "StoplossGuard",
-        "lookback_period_candles": 24,
-        "trade_limit": 4,
-        "stop_duration_candles": 4,
-        "only_per_pair": False
-    }
-]
+@property
+def protections(self):
+    return [
+        {
+            "method": "StoplossGuard",
+            "lookback_period_candles": 24,
+            "trade_limit": 4,
+            "stop_duration_candles": 4,
+            "only_per_pair": False
+        }
+    ]
 ```
 
 !!! Note
@@ -69,15 +75,17 @@ protections = [
 The below sample stops trading for 12 candles if max-drawdown is > 20% considering all pairs - with a minimum of `trade_limit` trades - within the last 48 candles. If desired, `lookback_period` and/or `stop_duration` can be used.
 
 ``` python
-protections = [
-      {
-        "method": "MaxDrawdown",
-        "lookback_period_candles": 48,
-        "trade_limit": 20,
-        "stop_duration_candles": 12,
-        "max_allowed_drawdown": 0.2
-      },
-]
+@property
+def protections(self):
+    return  [
+        {
+            "method": "MaxDrawdown",
+            "lookback_period_candles": 48,
+            "trade_limit": 20,
+            "stop_duration_candles": 12,
+            "max_allowed_drawdown": 0.2
+        },
+    ]
 ```
 
 #### Low Profit Pairs
@@ -88,15 +96,17 @@ If that ratio is below `required_profit`, that pair will be locked for `stop_dur
 The below example will stop trading a pair for 60 minutes if the pair does not have a required profit of 2% (and a minimum of 2 trades) within the last 6 candles.
 
 ``` python
-protections = [
-    {
-        "method": "LowProfitPairs",
-        "lookback_period_candles": 6,
-        "trade_limit": 2,
-        "stop_duration": 60,
-        "required_profit": 0.02
-    }
-]
+@property
+def protections(self):
+    return [
+        {
+            "method": "LowProfitPairs",
+            "lookback_period_candles": 6,
+            "trade_limit": 2,
+            "stop_duration": 60,
+            "required_profit": 0.02
+        }
+    ]
 ```
 
 #### Cooldown Period
@@ -106,12 +116,14 @@ protections = [
 The below example will stop trading a pair for 2 candles after closing a trade, allowing this pair to "cool down".
 
 ``` python
-protections = [
-    {
-        "method": "CooldownPeriod",
-        "stop_duration_candles": 2
-    }
-]
+@property
+def protections(self):
+    return  [
+        {
+            "method": "CooldownPeriod",
+            "stop_duration_candles": 2
+        }
+    ]
 ```
 
 !!! Note
@@ -136,39 +148,42 @@ from freqtrade.strategy import IStrategy
 
 class AwesomeStrategy(IStrategy)
     timeframe = '1h'
-    protections = [
-        {
-            "method": "CooldownPeriod",
-            "stop_duration_candles": 5
-        },
-        {
-            "method": "MaxDrawdown",
-            "lookback_period_candles": 48,
-            "trade_limit": 20,
-            "stop_duration_candles": 4,
-            "max_allowed_drawdown": 0.2
-        },
-        {
-            "method": "StoplossGuard",
-            "lookback_period_candles": 24,
-            "trade_limit": 4,
-            "stop_duration_candles": 2,
-            "only_per_pair": False
-        },
-        {
-            "method": "LowProfitPairs",
-            "lookback_period_candles": 6,
-            "trade_limit": 2,
-            "stop_duration_candles": 60,
-            "required_profit": 0.02
-        },
-        {
-            "method": "LowProfitPairs",
-            "lookback_period_candles": 24,
-            "trade_limit": 4,
-            "stop_duration_candles": 2,
-            "required_profit": 0.01
-        }
-    ]
+    
+    @property
+    def protections(self):
+        return [
+            {
+                "method": "CooldownPeriod",
+                "stop_duration_candles": 5
+            },
+            {
+                "method": "MaxDrawdown",
+                "lookback_period_candles": 48,
+                "trade_limit": 20,
+                "stop_duration_candles": 4,
+                "max_allowed_drawdown": 0.2
+            },
+            {
+                "method": "StoplossGuard",
+                "lookback_period_candles": 24,
+                "trade_limit": 4,
+                "stop_duration_candles": 2,
+                "only_per_pair": False
+            },
+            {
+                "method": "LowProfitPairs",
+                "lookback_period_candles": 6,
+                "trade_limit": 2,
+                "stop_duration_candles": 60,
+                "required_profit": 0.02
+            },
+            {
+                "method": "LowProfitPairs",
+                "lookback_period_candles": 24,
+                "trade_limit": 4,
+                "stop_duration_candles": 2,
+                "required_profit": 0.01
+            }
+        ]
     # ...
 ```

docs/index.md

@@ -36,7 +36,7 @@ Freqtrade is a crypto-currency algorithmic trading software developed in python
 
 Please read the [exchange specific notes](exchanges.md) to learn about eventual, special configurations needed for each exchange.
 
-- [X] [Binance](https://www.binance.com/) ([*Note for binance users](exchanges.md#blacklists))
+- [X] [Binance](https://www.binance.com/) ([*Note for binance users](docs/exchanges.md#binance-blacklist))
 - [X] [Bittrex](https://bittrex.com/)
 - [X] [FTX](https://ftx.com)
 - [X] [Kraken](https://kraken.com/)
@@ -47,6 +47,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual,
 Exchanges confirmed working by the community:
 
 - [X] [Bitvavo](https://bitvavo.com/)
+- [X] [Kucoin](https://www.kucoin.com/)
 
 ## Requirements
 
@@ -72,13 +73,9 @@ Alternatively
 
 ## Support
 
-### Help / Discord / Slack
+### Help / Discord
 
-For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join our slack channel.
-
-Please check out our [discord server](https://discord.gg/p7nuUNVfP7).
-
-You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw).
+For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join the Freqtrade [discord server](https://discord.gg/p7nuUNVfP7).
 
 ## Ready to try?
 

docs/installation.md

@@ -203,6 +203,8 @@ sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h
 ./configure --prefix=/usr/local
 make
 sudo make install
+# On debian based systems (debian, ubuntu, ...) - updating ldconfig might be necessary.
+sudo ldconfig  
 cd ..
 rm -rf ./ta-lib*
 ```

docs/requirements-docs.txt

@@ -1,4 +1,4 @@
-mkdocs==1.2.1
-mkdocs-material==7.1.8
+mkdocs==1.2.2
+mkdocs-material==7.2.4
 mdx_truly_sane_lists==1.2
 pymdown-extensions==8.2

docs/sql_cheatsheet.md

@@ -110,7 +110,7 @@ DELETE FROM trades WHERE id = 31;
 Freqtrade supports PostgreSQL by using SQLAlchemy, which supports multiple different database systems.
 
 Installation:
-`pip install psycopg2`
+`pip install psycopg2-binary`
 
 Usage:
 `... --db-url postgresql+psycopg2://<username>:<password>@localhost:5432/<database>`

docs/strategy-advanced.md

@@ -55,7 +55,7 @@ class AwesomeStrategy(IStrategy):
         dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
 
         # Obtain last available candle. Do not use current_time to look up latest candle, because 
-        # current_time points to curret incomplete candle whose data is not available.
+        # current_time points to current incomplete candle whose data is not available.
         last_candle = dataframe.iloc[-1].squeeze()
         # <...>
 
@@ -83,7 +83,7 @@ It is possible to define custom sell signals, indicating that specified position
 
 For example you could implement a 1:2 risk-reward ROI with `custom_sell()`.
 
-Using custom_sell() signals in place of stoplosses though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange.
+Using custom_sell() signals in place of stoploss though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange.
 
 !!! Note
     Returning a `string` or `True` from this method is equal to setting sell signal on a candle at specified time. This method is not called when sell signal is set already, or if sell signals are disabled (`use_sell_signal=False` or `sell_profit_only=True` while profit is below `sell_profit_offset`). `string` max length is 64 characters. Exceeding this limit will cause the message to be truncated to 64 characters.
@@ -114,6 +114,36 @@ class AwesomeStrategy(IStrategy):
 
 See [Dataframe access](#dataframe-access) for more information about dataframe use in strategy callbacks.
 
+## Buy Tag
+
+When your strategy has multiple buy signals, you can name the signal that triggered.
+Then you can access you buy signal on `custom_sell`
+
+```python
+def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+    dataframe.loc[
+        (
+            (dataframe['rsi'] < 35) &
+            (dataframe['volume'] > 0)
+        ),
+        ['buy', 'buy_tag']] = (1, 'buy_signal_rsi')
+
+    return dataframe
+
+def custom_sell(self, pair: str, trade: Trade, current_time: datetime, current_rate: float,
+                    current_profit: float, **kwargs):
+    dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
+    last_candle = dataframe.iloc[-1].squeeze()
+    if trade.buy_tag == 'buy_signal_rsi' and last_candle['rsi'] > 80:
+        return 'sell_signal_rsi'
+    return None
+
+```
+
+!!! Note
+    `buy_tag` is limited to 100 characters, remaining data will be truncated.
+
+
 ## Custom stoploss
 
 The stoploss price can only ever move upwards - if the stoploss value returned from `custom_stoploss` would result in a lower stoploss price than was previously set, it will be ignored. The traditional `stoploss` value serves as an absolute lower level and will be instated as the initial stoploss.
@@ -243,7 +273,7 @@ class AwesomeStrategy(IStrategy):
                         current_rate: float, current_profit: float, **kwargs) -> float:
 
         if current_profit < 0.04:
-            return -1 # return a value bigger than the inital stoploss to keep using the inital stoploss
+            return -1 # return a value bigger than the initial stoploss to keep using the initial stoploss
 
         # After reaching the desired offset, allow the stoploss to trail by half the profit
         desired_stoploss = current_profit / 2
@@ -327,6 +357,55 @@ See [Dataframe access](#dataframe-access) for more information about dataframe u
 
 ---
 
+## Custom order price rules
+
+By default, freqtrade use the orderbook to automatically set an order price([Relevant documentation](configuration.md#prices-used-for-orders)), you also have the option to create custom order prices based on your strategy.
+
+You can use this feature by creating a `custom_entry_price()` function in your strategy file to customize entry prices and `custom_exit_price()` for exits.
+
+!!! Note
+    If your custom pricing function return None or an invalid value, price will fall back to `proposed_rate`, which is based on the regular pricing configuration.
+
+### Custom order entry and exit price example
+
+``` python
+from datetime import datetime, timedelta, timezone
+from freqtrade.persistence import Trade
+
+class AwesomeStrategy(IStrategy):
+
+    # ... populate_* methods
+
+    def custom_entry_price(self, pair: str, current_time: datetime,
+                           proposed_rate, **kwargs) -> float:
+
+        dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair,
+                                                                timeframe=self.timeframe)
+        new_entryprice = dataframe['bollinger_10_lowerband'].iat[-1]
+        
+        return new_entryprice
+
+    def custom_exit_price(self, pair: str, trade: Trade,
+                          current_time: datetime, proposed_rate: float,
+                          current_profit: float, **kwargs) -> float:
+
+        dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair,
+                                                                timeframe=self.timeframe)
+        new_exitprice = dataframe['bollinger_10_upperband'].iat[-1]
+        
+        return new_exitprice
+
+```
+
+!!! Warning
+    Modifying entry and exit prices will only work for limit orders. Depending on the price chosen, this can result in a lot of unfilled orders. By default the maximum allowed distance between the current price and the custom price is 2%, this value can be changed in config with the `custom_price_max_distance_ratio` parameter.
+
+!!! Example
+    If the new_entryprice is 97, the proposed_rate is 100 and the `custom_price_max_distance_ratio` is set to 2%, The retained valid custom entry price will be 98.
+
+!!! Warning "No backtesting support"
+    Custom entry-prices are currently not supported during backtesting.
+
 ## Custom order timeout rules
 
 Simple, time-based order-timeouts can be configured either via strategy or in the configuration in the `unfilledtimeout` section.
@@ -454,7 +533,7 @@ class AwesomeStrategy(IStrategy):
     # ... populate_* methods
 
     def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float,
-                            time_in_force: str, **kwargs) -> bool:
+                            time_in_force: str, current_time: datetime, **kwargs) -> bool:
         """
         Called right before placing a buy order.
         Timing for this function is critical, so avoid doing heavy computations or
@@ -469,6 +548,7 @@ class AwesomeStrategy(IStrategy):
         :param amount: Amount in target (quote) currency that's going to be traded.
         :param rate: Rate that's going to be used when using limit orders
         :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled).
+        :param current_time: datetime object, containing the current datetime
         :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
         :return bool: When True is returned, then the buy-order is placed on the exchange.
             False aborts the process
@@ -490,7 +570,8 @@ class AwesomeStrategy(IStrategy):
     # ... populate_* methods
 
     def confirm_trade_exit(self, pair: str, trade: Trade, order_type: str, amount: float,
-                           rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool:
+                           rate: float, time_in_force: str, sell_reason: str,
+                           current_time: datetime, **kwargs) -> bool:
         """
         Called right before placing a regular sell order.
         Timing for this function is critical, so avoid doing heavy computations or
@@ -508,6 +589,7 @@ class AwesomeStrategy(IStrategy):
         :param sell_reason: Sell reason.
             Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss',
                            'sell_signal', 'force_sell', 'emergency_sell']
+        :param current_time: datetime object, containing the current datetime
         :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
         :return bool: When True is returned, then the sell-order is placed on the exchange.
             False aborts the process
@@ -521,6 +603,39 @@ class AwesomeStrategy(IStrategy):
 
 ```
 
+### Stake size management
+
+It is possible to manage your risk by reducing or increasing stake amount when placing a new trade.
+
+```python
+class AwesomeStrategy(IStrategy):
+    def custom_stake_amount(self, pair: str, current_time: datetime, current_rate: float,
+                            proposed_stake: float, min_stake: float, max_stake: float,
+                            **kwargs) -> float:
+
+        dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe)
+        current_candle = dataframe.iloc[-1].squeeze()
+
+        if current_candle['fastk_rsi_1h'] > current_candle['fastd_rsi_1h']:
+            if self.config['stake_amount'] == 'unlimited':
+                # Use entire available wallet during favorable conditions when in compounding mode.
+                return max_stake
+            else:
+                # Compound profits during favorable conditions instead of using a static stake.
+                return self.wallets.get_total_stake_amount() / self.config['max_open_trades']
+
+        # Use default stake amount.
+        return proposed_stake
+```
+
+Freqtrade will fall back to the `proposed_stake` value should your code raise an exception. The exception itself will be logged.
+
+!!! Tip
+    You do not _have_ to ensure that `min_stake <= returned_value <= max_stake`. Trades will succeed as the returned value will be clamped to supported range and this acton will be logged.
+
+!!! Tip
+    Returning `0` or `None` will prevent trades from being placed.
+
 ---
 
 ## Derived strategies

docs/strategy_analysis_example.md

@@ -130,6 +130,44 @@ trades = load_backtest_data(backtest_dir)
 trades.groupby("pair")["sell_reason"].value_counts()
 ```
 
+## Plotting daily profit / equity line
+
+
+```python
+# Plotting equity line (starting with 0 on day 1 and adding daily profit for each backtested day)
+
+from freqtrade.configuration import Configuration
+from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats
+import plotly.express as px
+import pandas as pd
+
+# strategy = 'SampleStrategy'
+# config = Configuration.from_files(["user_data/config.json"])
+# backtest_dir = config["user_data_dir"] / "backtest_results"
+
+stats = load_backtest_stats(backtest_dir)
+strategy_stats = stats['strategy'][strategy]
+
+dates = []
+profits = []
+for date_profit in strategy_stats['daily_profit']:
+    dates.append(date_profit[0])
+    profits.append(date_profit[1])
+
+equity = 0
+equity_daily = []
+for daily_profit in profits:
+    equity_daily.append(equity)
+    equity += float(daily_profit)
+
+
+df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily})
+
+fig = px.line(df, x="dates", y="equity_daily")
+fig.show()
+
+```
+
 ### Load live trading results into a pandas dataframe
 
 In case you did already some trading and want to analyze your performance
@@ -190,7 +228,7 @@ graph = generate_candlestick_graph(pair=pair,
 # Show graph inline
 # graph.show()
 
-# Render graph in a seperate window
+# Render graph in a separate window
 graph.show(renderer="browser")
 
 ```

docs/telegram-usage.md

@@ -245,10 +245,10 @@ current    max
 Return a summary of your profit/loss and performance.
 
 > **ROI:** Close trades  
->   ∙ `0.00485701 BTC (258.45%)`  
+>   ∙ `0.00485701 BTC (2.2%) (15.2 Σ%)`  
 >   ∙ `62.968 USD`  
 > **ROI:** All trades  
->   ∙ `0.00255280 BTC (143.43%)`  
+>   ∙ `0.00255280 BTC (1.5%) (6.43 Σ%)`  
 >   ∙ `33.095 EUR`  
 >  
 > **Total Trade Count:** `138`  
@@ -257,6 +257,10 @@ Return a summary of your profit/loss and performance.
 > **Avg. Duration:** `2:33:45`  
 > **Best Performing:** `PAY/BTC: 50.23%`  
 
+The relative profit of `1.2%` is the average profit per trade.  
+The relative profit of `15.2 Σ%` is be based on the starting capital - so in this case, the starting capital was `0.00485701 * 1.152 = 0.00738 BTC`.
+Starting capital is either taken from the `available_capital` setting, or calculated by using current wallet size - profits.
+
 ### /forcesell <trade_id>
 
 > **BITTREX:** Selling BTC/LTC with limit `0.01650000 (profit: ~-4.07%, -0.00008168)`

docs/utils.md

@@ -614,6 +614,42 @@ Show whitelist when using a [dynamic pairlist](plugins.md#pairlists).
 freqtrade test-pairlist --config config.json --quote USDT BTC
 ```
 
+## Webserver mode
+
+!!! Warning "Experimental"
+    Webserver mode is an experimental mode to increase backesting and strategy development productivity.
+    There may still be bugs - so if you happen to stumble across these, please report them as github issues, thanks.
+
+Run freqtrade in webserver mode.
+Freqtrade will start the webserver and allow FreqUI to start and control backtesting processes.
+This has the advantage that data will not be reloaded between backtesting runs (as long as timeframe and timerange remain identical).
+FreqUI will also show the backtesting results.
+
+```
+usage: freqtrade webserver [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
+                           [--userdir PATH]
+
+optional arguments:
+  -h, --help            show this help message and exit
+
+Common arguments:
+  -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
+  --logfile FILE        Log to the file specified. Special values are:
+                        'syslog', 'journald'. See the documentation for more
+                        details.
+  -V, --version         show program's version number and exit
+  -c PATH, --config PATH
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
+  -d PATH, --datadir PATH
+                        Path to directory with historical backtesting data.
+  --userdir PATH, --user-data-dir PATH
+                        Path to userdata directory.
+
+```
+
 ## List Hyperopt results
 
 You can list the hyperoptimization epochs the Hyperopt module evaluated previously with the `hyperopt-list` sub-command.
@@ -702,7 +738,8 @@ You can show the details of any hyperoptimization epoch previously evaluated by
 usage: freqtrade hyperopt-show [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                [-d PATH] [--userdir PATH] [--best]
                                [--profitable] [-n INT] [--print-json]
-                               [--hyperopt-filename PATH] [--no-header]
+                               [--hyperopt-filename FILENAME] [--no-header]
+                               [--disable-param-export]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -714,6 +751,8 @@ optional arguments:
                         Hyperopt result filename.Example: `--hyperopt-
                         filename=hyperopt_results_2020-09-27_16-20-48.pickle`
   --no-header           Do not print epoch details header.
+  --disable-param-export
+                        Disable automatic hyperopt parameter export.
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).

docs/webhook-config.md

@@ -83,6 +83,7 @@ Possible parameters are:
 * `fiat_currency`
 * `order_type`
 * `current_rate`
+* `buy_tag`
 
 ### Webhookbuycancel
 
@@ -100,6 +101,7 @@ Possible parameters are:
 * `fiat_currency`
 * `order_type`
 * `current_rate`
+* `buy_tag`
 
 ### Webhookbuyfill
 
@@ -115,6 +117,7 @@ Possible parameters are:
 * `stake_amount`
 * `stake_currency`
 * `fiat_currency`
+* `buy_tag`
 
 ### Webhooksell
 

docs/windows_installation.md

@@ -23,7 +23,7 @@ git clone https://github.com/freqtrade/freqtrade.git
 
 Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows).
 
-As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial pre-compiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.20‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version).
+As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial pre-compiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which need to be downloaded and installed using `pip install TA_Lib-0.4.21-cp38-cp38-win_amd64.whl` (make sure to use the version matching your python version).
 
 Freqtrade provides these dependencies for the latest 2 Python versions (3.7 and 3.8) and for 64bit Windows.
 Other versions must be downloaded from the above link.

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2021.6'
+__version__ = '2021.8'
 
 if __version__ == 'develop':
 

freqtrade/commands/__init__.py

@@ -20,3 +20,4 @@ from freqtrade.commands.optimize_commands import start_backtesting, start_edge,
 from freqtrade.commands.pairlist_commands import start_test_pairlist
 from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit
 from freqtrade.commands.trade_commands import start_trading
+from freqtrade.commands.webserver_commands import start_webserver

freqtrade/commands/arguments.py

@@ -16,6 +16,8 @@ ARGS_STRATEGY = ["strategy", "strategy_path"]
 
 ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"]
 
+ARGS_WEBSERVER: List[str] = []
+
 ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
                         "max_open_trades", "stake_amount", "fee", "pairs"]
 
@@ -29,7 +31,7 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
                                         "epochs", "spaces", "print_all",
                                         "print_colorized", "print_json", "hyperopt_jobs",
                                         "hyperopt_random_state", "hyperopt_min_trades",
-                                        "hyperopt_loss"]
+                                        "hyperopt_loss", "disableparamexport"]
 
 ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]
 
@@ -85,7 +87,8 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable",
                       "hyperoptexportfilename", "export_csv"]
 
 ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index",
-                      "print_json", "hyperoptexportfilename", "hyperopt_show_no_header"]
+                      "print_json", "hyperoptexportfilename", "hyperopt_show_no_header",
+                      "disableparamexport"]
 
 NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
                     "list-markets", "list-pairs", "list-strategies", "list-data",
@@ -175,7 +178,8 @@ class Arguments:
                                         start_list_markets, start_list_strategies,
                                         start_list_timeframes, start_new_config, start_new_hyperopt,
                                         start_new_strategy, start_plot_dataframe, start_plot_profit,
-                                        start_show_trades, start_test_pairlist, start_trading)
+                                        start_show_trades, start_test_pairlist, start_trading,
+                                        start_webserver)
 
         subparsers = self.parser.add_subparsers(dest='command',
                                                 # Use custom message when no subhandler is added
@@ -383,3 +387,9 @@ class Arguments:
         )
         plot_profit_cmd.set_defaults(func=start_plot_profit)
         self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)
+
+        # Add webserver subcommand
+        webserver_cmd = subparsers.add_parser('webserver', help='Webserver module.',
+                                              parents=[_common_parser])
+        webserver_cmd.set_defaults(func=start_webserver)
+        self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd)

freqtrade/commands/build_config_commands.py

@@ -193,7 +193,7 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None:
         selections['exchange'] = render_template(
             templatefile=f"subtemplates/exchange_{exchange_template}.j2",
             arguments=selections
-            )
+        )
     except TemplateNotFound:
         selections['exchange'] = render_template(
             templatefile="subtemplates/exchange_generic.j2",

freqtrade/commands/cli_options.py

@@ -162,7 +162,7 @@ AVAILABLE_CLI_OPTIONS = {
         'Please note that ticker-interval needs to be set either in config '
         'or via command line. When using this together with `--export trades`, '
         'the strategy-name is injected into the filename '
-        '(so `backtest-data.json` becomes `backtest-data-DefaultStrategy.json`',
+        '(so `backtest-data.json` becomes `backtest-data-SampleStrategy.json`',
         nargs='+',
     ),
     "export": Arg(
@@ -178,6 +178,11 @@ AVAILABLE_CLI_OPTIONS = {
         'Example: `--export-filename=user_data/backtest_results/backtest_today.json`',
         metavar='PATH',
     ),
+    "disableparamexport": Arg(
+        '--disable-param-export',
+        help="Disable automatic hyperopt parameter export.",
+        action='store_true',
+    ),
     "fee": Arg(
         '--fee',
         help='Specify fee ratio. Will be applied twice (on trade entry and exit).',
@@ -213,7 +218,7 @@ AVAILABLE_CLI_OPTIONS = {
     "spaces": Arg(
         '--spaces',
         help='Specify which parameters to hyperopt. Space-separated list.',
-        choices=['all', 'buy', 'sell', 'roi', 'stoploss', 'trailing', 'default'],
+        choices=['all', 'buy', 'sell', 'roi', 'stoploss', 'trailing', 'protection', 'default'],
         nargs='+',
         default='default',
     ),

freqtrade/commands/data_commands.py

@@ -48,7 +48,8 @@ def start_download_data(args: Dict[str, Any]) -> None:
     # Init exchange
     exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
     # Manual validations of relevant settings
-    exchange.validate_pairs(config['pairs'])
+    if not config['exchange'].get('skip_pair_validation', False):
+        exchange.validate_pairs(config['pairs'])
     expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
 
     logger.info(f"About to download pairs: {expanded_pairs}, "

freqtrade/commands/deploy_commands.py

@@ -38,15 +38,15 @@ def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: st
     indicators = render_template_with_fallback(
         templatefile=f"subtemplates/indicators_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/indicators_{fallback}.j2",
-        )
+    )
     buy_trend = render_template_with_fallback(
         templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/buy_trend_{fallback}.j2",
-        )
+    )
     sell_trend = render_template_with_fallback(
         templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/sell_trend_{fallback}.j2",
-        )
+    )
     plot_config = render_template_with_fallback(
         templatefile=f"subtemplates/plot_config_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/plot_config_{fallback}.j2",
@@ -74,8 +74,6 @@ def start_new_strategy(args: Dict[str, Any]) -> None:
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
     if "strategy" in args and args["strategy"]:
-        if args["strategy"] == "DefaultStrategy":
-            raise OperationalException("DefaultStrategy is not allowed as name.")
 
         new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py')
 
@@ -97,19 +95,19 @@ def deploy_new_hyperopt(hyperopt_name: str, hyperopt_path: Path, subtemplate: st
     buy_guards = render_template_with_fallback(
         templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/hyperopt_buy_guards_{fallback}.j2",
-        )
+    )
     sell_guards = render_template_with_fallback(
         templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/hyperopt_sell_guards_{fallback}.j2",
-        )
+    )
     buy_space = render_template_with_fallback(
         templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/hyperopt_buy_space_{fallback}.j2",
-        )
+    )
     sell_space = render_template_with_fallback(
         templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",
         templatefallbackfile=f"subtemplates/hyperopt_sell_space_{fallback}.j2",
-        )
+    )
 
     strategy_text = render_template(templatefile='base_hyperopt.py.j2',
                                     arguments={"hyperopt": hyperopt_name,
@@ -128,8 +126,6 @@ def start_new_hyperopt(args: Dict[str, Any]) -> None:
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
     if 'hyperopt' in args and args['hyperopt']:
-        if args['hyperopt'] == 'DefaultHyperopt':
-            raise OperationalException("DefaultHyperopt is not allowed as name.")
 
         new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args['hyperopt'] + '.py')
 

freqtrade/commands/hyperopt_commands.py

@@ -1,6 +1,6 @@
 import logging
 from operator import itemgetter
-from typing import Any, Dict, List
+from typing import Any, Dict
 
 from colorama import init as colorama_init
 
@@ -28,30 +28,12 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     no_details = config.get('hyperopt_list_no_details', False)
     no_header = False
 
-    filteroptions = {
-        'only_best': config.get('hyperopt_list_best', False),
-        'only_profitable': config.get('hyperopt_list_profitable', False),
-        'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
-        'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
-        'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
-        'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
-        'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
-        'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
-        'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
-        'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
-        'filter_min_objective': config.get('hyperopt_list_min_objective', None),
-        'filter_max_objective': config.get('hyperopt_list_max_objective', None),
-    }
-
     results_file = get_latest_hyperopt_file(
         config['user_data_dir'] / 'hyperopt_results',
         config.get('hyperoptexportfilename'))
 
     # Previous evaluations
-    epochs = HyperoptTools.load_previous_results(results_file)
-    total_epochs = len(epochs)
-
-    epochs = hyperopt_filter_epochs(epochs, filteroptions)
+    epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
 
     if print_colorized:
         colorama_init(autoreset=True)
@@ -59,7 +41,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     if not export_csv:
         try:
             print(HyperoptTools.get_result_table(config, epochs, total_epochs,
-                                                 not filteroptions['only_best'],
+                                                 not config.get('hyperopt_list_best', False),
                                                  print_colorized, 0))
         except KeyboardInterrupt:
             print('User interrupted..')
@@ -71,7 +53,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
 
     if epochs and export_csv:
         HyperoptTools.export_csv_file(
-            config, epochs, total_epochs, not filteroptions['only_best'], export_csv
+            config, epochs, total_epochs, not config.get('hyperopt_list_best', False), export_csv
         )
 
 
@@ -91,26 +73,9 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
 
     n = config.get('hyperopt_show_index', -1)
 
-    filteroptions = {
-        'only_best': config.get('hyperopt_list_best', False),
-        'only_profitable': config.get('hyperopt_list_profitable', False),
-        'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
-        'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
-        'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
-        'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
-        'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
-        'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
-        'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
-        'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
-        'filter_min_objective': config.get('hyperopt_list_min_objective', None),
-        'filter_max_objective': config.get('hyperopt_list_max_objective', None)
-    }
-
     # Previous evaluations
-    epochs = HyperoptTools.load_previous_results(results_file)
-    total_epochs = len(epochs)
+    epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
 
-    epochs = hyperopt_filter_epochs(epochs, filteroptions)
     filtered_epochs = len(epochs)
 
     if n > filtered_epochs:
@@ -129,143 +94,11 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
 
         metrics = val['results_metrics']
         if 'strategy_name' in metrics:
-            show_backtest_result(metrics['strategy_name'], metrics,
+            strategy_name = metrics['strategy_name']
+            show_backtest_result(strategy_name, metrics,
                                  metrics['stake_currency'])
 
+            HyperoptTools.try_export_params(config, strategy_name, val)
+
         HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header,
                                          header_str="Epoch details")
-
-
-def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
-    """
-    Filter our items from the list of hyperopt results
-    TODO: after 2021.5 remove all "legacy" mode queries.
-    """
-    if filteroptions['only_best']:
-        epochs = [x for x in epochs if x['is_best']]
-    if filteroptions['only_profitable']:
-        epochs = [x for x in epochs if x['results_metrics'].get(
-            'profit', x['results_metrics'].get('profit_total', 0)) > 0]
-
-    epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
-
-    epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions)
-
-    epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions)
-
-    epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions)
-
-    logger.info(f"{len(epochs)} " +
-                ("best " if filteroptions['only_best'] else "") +
-                ("profitable " if filteroptions['only_profitable'] else "") +
-                "epochs found.")
-    return epochs
-
-
-def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int):
-    """
-    Filter epochs with trade-counts > trades
-    """
-    return [
-        x for x in epochs
-        if x['results_metrics'].get(
-            'trade_count', x['results_metrics'].get('total_trades', 0)
-        ) > trade_count
-    ]
-
-
-def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
-
-    if filteroptions['filter_min_trades'] > 0:
-        epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades'])
-
-    if filteroptions['filter_max_trades'] > 0:
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'trade_count', x['results_metrics'].get('total_trades')
-                ) < filteroptions['filter_max_trades']
-        ]
-    return epochs
-
-
-def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
-
-    def get_duration_value(x):
-        # Duration in minutes ...
-        if 'duration' in x['results_metrics']:
-            return x['results_metrics']['duration']
-        else:
-            # New mode
-            if 'holding_avg_s' in x['results_metrics']:
-                avg = x['results_metrics']['holding_avg_s']
-                return avg // 60
-            raise OperationalException(
-                "Holding-average not available. Please omit the filter on average time, "
-                "or rerun hyperopt with this version")
-
-    if filteroptions['filter_min_avg_time'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if get_duration_value(x) > filteroptions['filter_min_avg_time']
-        ]
-    if filteroptions['filter_max_avg_time'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if get_duration_value(x) < filteroptions['filter_max_avg_time']
-        ]
-
-    return epochs
-
-
-def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
-
-    if filteroptions['filter_min_avg_profit'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
-            ) > filteroptions['filter_min_avg_profit']
-        ]
-    if filteroptions['filter_max_avg_profit'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
-                ) < filteroptions['filter_max_avg_profit']
-        ]
-    if filteroptions['filter_min_total_profit'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'profit', x['results_metrics'].get('profit_total_abs', 0)
-                ) > filteroptions['filter_min_total_profit']
-        ]
-    if filteroptions['filter_max_total_profit'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-        epochs = [
-            x for x in epochs
-            if x['results_metrics'].get(
-                'profit', x['results_metrics'].get('profit_total_abs', 0)
-                ) < filteroptions['filter_max_total_profit']
-        ]
-    return epochs
-
-
-def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
-
-    if filteroptions['filter_min_objective'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-
-        epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
-    if filteroptions['filter_max_objective'] is not None:
-        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
-
-        epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
-
-    return epochs

freqtrade/commands/list_commands.py

@@ -14,7 +14,7 @@ from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import market_is_active, validate_exchanges
-from freqtrade.misc import plural
+from freqtrade.misc import parse_db_uri_for_logging, plural
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 
 
@@ -225,7 +225,7 @@ def start_show_trades(args: Dict[str, Any]) -> None:
     if 'db_url' not in config:
         raise OperationalException("--db-url is required for this command.")
 
-    logger.info(f'Using DB: "{config["db_url"]}"')
+    logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"')
     init_db(config['db_url'], clean_open_orders=False)
     tfilter = []
 

freqtrade/commands/webserver_commands.py

@@ -0,0 +1,15 @@
+from typing import Any, Dict
+
+from freqtrade.enums import RunMode
+
+
+def start_webserver(args: Dict[str, Any]) -> None:
+    """
+    Main entry point for webserver mode
+    """
+    from freqtrade.configuration import Configuration
+    from freqtrade.rpc.api_server import ApiServer
+
+    # Initialize configuration
+    config = Configuration(args, RunMode.WEBSERVER).get_config()
+    ApiServer(config, standalone=True)

freqtrade/configuration/check_exchange.py

@@ -51,10 +51,10 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
 
     if not is_exchange_known_ccxt(exchange):
         raise OperationalException(
-                f'Exchange "{exchange}" is not known to the ccxt library '
-                f'and therefore not available for the bot.\n'
-                f'The following exchanges are available for Freqtrade: '
-                f'{", ".join(available_exchanges())}'
+            f'Exchange "{exchange}" is not known to the ccxt library '
+            f'and therefore not available for the bot.\n'
+            f'The following exchanges are available for Freqtrade: '
+            f'{", ".join(available_exchanges())}'
         )
 
     valid, reason = validate_exchange(exchange)

freqtrade/configuration/config_validation.py

@@ -79,6 +79,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None:
     _validate_whitelist(conf)
     _validate_protections(conf)
     _validate_unlimited_amount(conf)
+    _validate_ask_orderbook(conf)
 
     # validate configuration before returning
     logger.info('Validating configuration ...')
@@ -114,7 +115,7 @@ def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None:
     if conf.get('stoploss') == 0.0:
         raise OperationalException(
             'The config stoploss needs to be different from 0 to avoid problems with sell orders.'
-            )
+        )
     # Skip if trailing stoploss is not activated
     if not conf.get('trailing_stop', False):
         return
@@ -149,7 +150,7 @@ def _validate_edge(conf: Dict[str, Any]) -> None:
     if not conf.get('edge', {}).get('enabled'):
         return
 
-    if not conf.get('ask_strategy', {}).get('use_sell_signal', True):
+    if not conf.get('use_sell_signal', True):
         raise OperationalException(
             "Edge requires `use_sell_signal` to be True, otherwise no sells will happen."
         )
@@ -179,10 +180,30 @@ def _validate_protections(conf: Dict[str, Any]) -> None:
             raise OperationalException(
                 "Protections must specify either `stop_duration` or `stop_duration_candles`.\n"
                 f"Please fix the protection {prot.get('method')}"
-                )
+            )
 
         if ('lookback_period' in prot and 'lookback_period_candles' in prot):
             raise OperationalException(
                 "Protections must specify either `lookback_period` or `lookback_period_candles`.\n"
                 f"Please fix the protection {prot.get('method')}"
             )
+
+
+def _validate_ask_orderbook(conf: Dict[str, Any]) -> None:
+    ask_strategy = conf.get('ask_strategy', {})
+    ob_min = ask_strategy.get('order_book_min')
+    ob_max = ask_strategy.get('order_book_max')
+    if ob_min is not None and ob_max is not None and ask_strategy.get('use_order_book'):
+        if ob_min != ob_max:
+            raise OperationalException(
+                "Using order_book_max != order_book_min in ask_strategy is no longer supported."
+                "Please pick one value and use `order_book_top` in the future."
+            )
+        else:
+            # Move value to order_book_top
+            ask_strategy['order_book_top'] = ob_min
+            logger.warning(
+                "DEPRECATED: "
+                "Please use `order_book_top` instead of `order_book_min` and `order_book_max` "
+                "for your `ask_strategy` configuration."
+            )

freqtrade/configuration/configuration.py

@@ -11,11 +11,12 @@ from freqtrade import constants
 from freqtrade.configuration.check_exchange import check_exchange
 from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
 from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
+from freqtrade.configuration.environment_vars import enironment_vars_to_dict
 from freqtrade.configuration.load_config import load_config_file, load_file
 from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.loggers import setup_logging
-from freqtrade.misc import deep_merge_dicts
+from freqtrade.misc import deep_merge_dicts, parse_db_uri_for_logging
 
 
 logger = logging.getLogger(__name__)
@@ -72,6 +73,11 @@ class Configuration:
             # Merge config options, overwriting old values
             config = deep_merge_dicts(load_config_file(path), config)
 
+        # Load environment variables
+        env_data = enironment_vars_to_dict()
+        config = deep_merge_dicts(env_data, config)
+
+        config['config_files'] = files
         # Normalize config
         if 'internals' not in config:
             config['internals'] = {}
@@ -144,7 +150,7 @@ class Configuration:
                 config['db_url'] = constants.DEFAULT_DB_PROD_URL
             logger.info('Dry run is disabled')
 
-        logger.info(f'Using DB: "{config["db_url"]}"')
+        logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"')
 
     def _process_common_options(self, config: Dict[str, Any]) -> None:
 
@@ -260,6 +266,8 @@ class Configuration:
         self._args_to_config(config, argname='export',
                              logstring='Parameter --export detected: {} ...')
 
+        self._args_to_config(config, argname='disableparamexport',
+                             logstring='Parameter --disableparamexport detected: {} ...')
         # Edge section:
         if 'stoploss_range' in self.args and self.args["stoploss_range"]:
             txt_range = eval(self.args["stoploss_range"])

freqtrade/configuration/deprecated_settings.py

@@ -3,7 +3,7 @@ Functions to handle deprecated settings
 """
 
 import logging
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 from freqtrade.exceptions import OperationalException
 
@@ -12,23 +12,24 @@ logger = logging.getLogger(__name__)
 
 
 def check_conflicting_settings(config: Dict[str, Any],
-                               section1: str, name1: str,
-                               section2: str, name2: str) -> None:
-    section1_config = config.get(section1, {})
-    section2_config = config.get(section2, {})
-    if name1 in section1_config and name2 in section2_config:
+                               section_old: str, name_old: str,
+                               section_new: Optional[str], name_new: str) -> None:
+    section_new_config = config.get(section_new, {}) if section_new else config
+    section_old_config = config.get(section_old, {})
+    if name_new in section_new_config and name_old in section_old_config:
+        new_name = f"{section_new}.{name_new}" if section_new else f"{name_new}"
         raise OperationalException(
-            f"Conflicting settings `{section1}.{name1}` and `{section2}.{name2}` "
+            f"Conflicting settings `{new_name}` and `{section_old}.{name_old}` "
             "(DEPRECATED) detected in the configuration file. "
             "This deprecated setting will be removed in the next versions of Freqtrade. "
-            f"Please delete it from your configuration and use the `{section1}.{name1}` "
+            f"Please delete it from your configuration and use the `{new_name}` "
             "setting instead."
         )
 
 
 def process_removed_setting(config: Dict[str, Any],
                             section1: str, name1: str,
-                            section2: str, name2: str) -> None:
+                            section2: Optional[str], name2: str) -> None:
     """
     :param section1: Removed section
     :param name1: Removed setting name
@@ -37,27 +38,32 @@ def process_removed_setting(config: Dict[str, Any],
     """
     section1_config = config.get(section1, {})
     if name1 in section1_config:
+        section_2 = f"{section2}.{name2}" if section2 else f"{name2}"
         raise OperationalException(
-            f"Setting `{section1}.{name1}` has been moved to `{section2}.{name2}. "
-            f"Please delete it from your configuration and use the `{section2}.{name2}` "
+            f"Setting `{section1}.{name1}` has been moved to `{section_2}. "
+            f"Please delete it from your configuration and use the `{section_2}` "
             "setting instead."
         )
 
 
 def process_deprecated_setting(config: Dict[str, Any],
-                               section1: str, name1: str,
-                               section2: str, name2: str) -> None:
-    section2_config = config.get(section2, {})
+                               section_old: str, name_old: str,
+                               section_new: Optional[str], name_new: str
+                               ) -> None:
+    check_conflicting_settings(config, section_old, name_old, section_new, name_new)
+    section_old_config = config.get(section_old, {})
 
-    if name2 in section2_config:
+    if name_old in section_old_config:
+        section_2 = f"{section_new}.{name_new}" if section_new else f"{name_new}"
         logger.warning(
             "DEPRECATED: "
-            f"The `{section2}.{name2}` setting is deprecated and "
+            f"The `{section_old}.{name_old}` setting is deprecated and "
             "will be removed in the next versions of Freqtrade. "
-            f"Please use the `{section1}.{name1}` setting in your configuration instead."
+            f"Please use the `{section_2}` setting in your configuration instead."
         )
-        section1_config = config.get(section1, {})
-        section1_config[name1] = section2_config[name2]
+
+        section_new_config = config.get(section_new, {}) if section_new else config
+        section_new_config[name_new] = section_old_config[name_old]
 
 
 def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
@@ -65,15 +71,24 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
     # Kept for future deprecated / moved settings
     # check_conflicting_settings(config, 'ask_strategy', 'use_sell_signal',
     #                            'experimental', 'use_sell_signal')
-    # process_deprecated_setting(config, 'ask_strategy', 'use_sell_signal',
-    #                            'experimental', 'use_sell_signal')
+    process_deprecated_setting(config, 'ask_strategy', 'use_sell_signal',
+                               None, 'use_sell_signal')
+    process_deprecated_setting(config, 'ask_strategy', 'sell_profit_only',
+                               None, 'sell_profit_only')
+    process_deprecated_setting(config, 'ask_strategy', 'sell_profit_offset',
+                               None, 'sell_profit_offset')
+    process_deprecated_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal',
+                               None, 'ignore_roi_if_buy_signal')
+    process_deprecated_setting(config, 'ask_strategy', 'ignore_buying_expired_candle_after',
+                               None, 'ignore_buying_expired_candle_after')
 
+    # Legacy way - having them in experimental ...
     process_removed_setting(config, 'experimental', 'use_sell_signal',
-                            'ask_strategy', 'use_sell_signal')
+                            None, 'use_sell_signal')
     process_removed_setting(config, 'experimental', 'sell_profit_only',
-                            'ask_strategy', 'sell_profit_only')
+                            None, 'sell_profit_only')
     process_removed_setting(config, 'experimental', 'ignore_roi_if_buy_signal',
-                            'ask_strategy', 'ignore_roi_if_buy_signal')
+                            None, 'ignore_roi_if_buy_signal')
 
     if (config.get('edge', {}).get('enabled', False)
        and 'capital_available_percentage' in config.get('edge', {})):
@@ -93,5 +108,8 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
             raise OperationalException(
                 "Both 'timeframe' and 'ticker_interval' detected."
                 "Please remove 'ticker_interval' from your configuration to continue operating."
-                )
+            )
         config['timeframe'] = config['ticker_interval']
+
+    if 'protections' in config:
+        logger.warning("DEPRECATED: Setting 'protections' in the configuration is deprecated.")

freqtrade/configuration/environment_vars.py

@@ -0,0 +1,54 @@
+import logging
+import os
+from typing import Any, Dict
+
+from freqtrade.constants import ENV_VAR_PREFIX
+from freqtrade.misc import deep_merge_dicts
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_var_typed(val):
+    try:
+        return int(val)
+    except ValueError:
+        try:
+            return float(val)
+        except ValueError:
+            if val.lower() in ('t', 'true'):
+                return True
+            elif val.lower() in ('f', 'false'):
+                return False
+    # keep as string
+    return val
+
+
+def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]:
+    """
+    Environment variables must be prefixed with FREQTRADE.
+    FREQTRADE__{section}__{key}
+    :param env_dict: Dictionary to validate - usually os.environ
+    :param prefix: Prefix to consider (usually FREQTRADE__)
+    :return: Nested dict based on available and relevant variables.
+    """
+    relevant_vars: Dict[str, Any] = {}
+
+    for env_var, val in sorted(env_dict.items()):
+        if env_var.startswith(prefix):
+            logger.info(f"Loading variable '{env_var}'")
+            key = env_var.replace(prefix, '')
+            for k in reversed(key.split('__')):
+                val = {k.lower(): get_var_typed(val) if type(val) != dict else val}
+            relevant_vars = deep_merge_dicts(val, relevant_vars)
+
+    return relevant_vars
+
+
+def enironment_vars_to_dict() -> Dict[str, Any]:
+    """
+    Read environment variables and return a nested dict for relevant variables
+    Relevant variables must follow the FREQTRADE__{section}__{key} pattern
+    :return: Nested dict based on available and relevant variables.
+    """
+    return flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX)

freqtrade/constants.py

@@ -26,9 +26,9 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
                          'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily',
                          'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily']
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
-                       'AgeFilter', 'PerformanceFilter', 'PrecisionFilter',
-                       'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter',
-                       'SpreadFilter', 'VolatilityFilter']
+                       'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
+                       'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
+                       'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
 AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
 AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
 DRY_RUN_WALLET = 1000
@@ -40,12 +40,16 @@ DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume']
 DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost']
 
 LAST_BT_RESULT_FN = '.last_result.json'
+FTHYPT_FILEVERSION = 'fthypt_fileversion'
 
 USERPATH_HYPEROPTS = 'hyperopts'
 USERPATH_STRATEGIES = 'strategies'
 USERPATH_NOTEBOOKS = 'notebooks'
 
 TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent']
+ENV_VAR_PREFIX = 'FREQTRADE__'
+
+NON_OPEN_EXCHANGE_STATES = ('cancelled', 'canceled', 'closed', 'expired')
 
 
 # Define decimals per coin for outputs
@@ -112,6 +116,10 @@ CONF_SCHEMA = {
             'maximum': 1,
             'default': 0.99
         },
+        'available_capital': {
+            'type': 'number',
+            'minimum': 0,
+        },
         'amend_last_stake_amount': {'type': 'boolean', 'default': False},
         'last_stake_amount_min_ratio': {
             'type': 'number', 'minimum': 0.0, 'maximum': 1.0, 'default': 0.5
@@ -134,6 +142,11 @@ CONF_SCHEMA = {
         'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1},
         'trailing_stop_positive_offset': {'type': 'number', 'minimum': 0, 'maximum': 1},
         'trailing_only_offset_is_reached': {'type': 'boolean'},
+        'use_sell_signal': {'type': 'boolean'},
+        'sell_profit_only': {'type': 'boolean'},
+        'sell_profit_offset': {'type': 'number'},
+        'ignore_roi_if_buy_signal': {'type': 'boolean'},
+        'ignore_buying_expired_candle_after': {'type': 'number'},
         'bot_name': {'type': 'string'},
         'unfilledtimeout': {
             'type': 'object',
@@ -154,7 +167,7 @@ CONF_SCHEMA = {
                 },
                 'price_side': {'type': 'string', 'enum': ORDERBOOK_SIDES, 'default': 'bid'},
                 'use_order_book': {'type': 'boolean'},
-                'order_book_top': {'type': 'integer', 'maximum': 20, 'minimum': 1},
+                'order_book_top': {'type': 'integer', 'minimum': 1, 'maximum': 50, },
                 'check_depth_of_market': {
                     'type': 'object',
                     'properties': {
@@ -163,7 +176,7 @@ CONF_SCHEMA = {
                     }
                 },
             },
-            'required': ['ask_last_balance']
+            'required': ['price_side']
         },
         'ask_strategy': {
             'type': 'object',
@@ -176,13 +189,12 @@ CONF_SCHEMA = {
                     'exclusiveMaximum': False,
                 },
                 'use_order_book': {'type': 'boolean'},
-                'order_book_min': {'type': 'integer', 'minimum': 1},
-                'order_book_max': {'type': 'integer', 'minimum': 1, 'maximum': 50},
-                'use_sell_signal': {'type': 'boolean'},
-                'sell_profit_only': {'type': 'boolean'},
-                'sell_profit_offset': {'type': 'number'},
-                'ignore_roi_if_buy_signal': {'type': 'boolean'}
-            }
+                'order_book_top': {'type': 'integer', 'minimum': 1, 'maximum': 50, },
+            },
+            'required': ['price_side']
+        },
+        'custom_price_max_distance_ratio': {
+           'type': 'number', 'minimum': 0.0
         },
         'order_types': {
             'type': 'object',
@@ -273,7 +285,7 @@ CONF_SCHEMA = {
                             'type': 'string',
                             'enum': TELEGRAM_SETTING_OPTIONS,
                             'default': 'off'
-                            },
+                        },
                     }
                 },
                 'reload': {'type': 'boolean'},
@@ -311,6 +323,7 @@ CONF_SCHEMA = {
         },
         'db_url': {'type': 'string'},
         'export': {'type': 'string', 'enum': EXPORT_OPTIONS, 'default': 'trades'},
+        'disableparamexport': {'type': 'boolean'},
         'initial_state': {'type': 'string', 'enum': ['running', 'stopped']},
         'forcebuy_enable': {'type': 'boolean'},
         'disable_dataframe_checks': {'type': 'boolean'},

freqtrade/data/btanalysis.py

@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
 BT_DATA_COLUMNS_OLD = ["pair", "profit_percent", "open_date", "close_date", "index",
                        "trade_duration", "open_rate", "close_rate", "open_at_end", "sell_reason"]
 
-# Mid-term format, crated by BacktestResult Named Tuple
+# Mid-term format, created by BacktestResult Named Tuple
 BT_DATA_COLUMNS_MID = ['pair', 'profit_percent', 'open_date', 'close_date', 'trade_duration',
                        'open_rate', 'close_rate', 'open_at_end', 'sell_reason', 'fee_open',
                        'fee_close', 'amount', 'profit_abs', 'profit_ratio']
@@ -30,7 +30,7 @@ BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
                    'fee_open', 'fee_close', 'trade_duration',
                    'profit_ratio', 'profit_abs', 'sell_reason',
                    'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
-                   'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', ]
+                   'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'buy_tag']
 
 
 def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str:

freqtrade/data/converter.py

@@ -242,7 +242,7 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to:
     :param config: Config dictionary
     :param convert_from: Source format
     :param convert_to: Target format
-    :param erase: Erase souce data (does not apply if source and target format are identical)
+    :param erase: Erase source data (does not apply if source and target format are identical)
     """
     from freqtrade.data.history.idatahandler import get_datahandler
     src = get_datahandler(config['datadir'], convert_from)
@@ -267,7 +267,7 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to:
     :param config: Config dictionary
     :param convert_from: Source format
     :param convert_to: Target format
-    :param erase: Erase souce data (does not apply if source and target format are identical)
+    :param erase: Erase source data (does not apply if source and target format are identical)
     """
     from freqtrade.data.history.idatahandler import get_datahandler
     src = get_datahandler(config['datadir'], convert_from)

freqtrade/data/dataprovider.py

@@ -10,11 +10,12 @@ from typing import Any, Dict, List, Optional, Tuple
 
 from pandas import DataFrame
 
+from freqtrade.configuration import TimeRange
 from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe
 from freqtrade.data.history import load_pair_history
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import ExchangeError, OperationalException
-from freqtrade.exchange import Exchange
+from freqtrade.exchange import Exchange, timeframe_to_seconds
 
 
 logger = logging.getLogger(__name__)
@@ -31,6 +32,7 @@ class DataProvider:
         self._pairlists = pairlists
         self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
         self.__slice_index: Optional[int] = None
+        self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {}
 
     def _set_dataframe_max_index(self, limit_index: int):
         """
@@ -62,11 +64,22 @@ class DataProvider:
         :param pair: pair to get the data for
         :param timeframe: timeframe to get data for
         """
-        return load_pair_history(pair=pair,
-                                 timeframe=timeframe or self._config['timeframe'],
-                                 datadir=self._config['datadir'],
-                                 data_format=self._config.get('dataformat_ohlcv', 'json')
-                                 )
+        saved_pair = (pair, str(timeframe))
+        if saved_pair not in self.__cached_pairs_backtesting:
+            timerange = TimeRange.parse_timerange(None if self._config.get(
+                'timerange') is None else str(self._config.get('timerange')))
+            # Move informative start time respecting startup_candle_count
+            timerange.subtract_start(
+                timeframe_to_seconds(str(timeframe)) * self._config.get('startup_candle_count', 0)
+            )
+            self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
+                pair=pair,
+                timeframe=timeframe or self._config['timeframe'],
+                datadir=self._config['datadir'],
+                timerange=timerange,
+                data_format=self._config.get('dataformat_ohlcv', 'json')
+            )
+        return self.__cached_pairs_backtesting[saved_pair].copy()
 
     def get_pair_dataframe(self, pair: str, timeframe: str = None) -> DataFrame:
         """

freqtrade/data/history/history_utils.py

@@ -117,10 +117,11 @@ def refresh_data(datadir: Path,
     :param timerange: Limit data to be loaded to this timerange
     """
     data_handler = get_datahandler(datadir, data_format)
-    for pair in pairs:
-        _download_pair_history(pair=pair, timeframe=timeframe,
-                               datadir=datadir, timerange=timerange,
-                               exchange=exchange, data_handler=data_handler)
+    for idx, pair in enumerate(pairs):
+        process = f'{idx}/{len(pairs)}'
+        _download_pair_history(pair=pair, process=process,
+                               timeframe=timeframe, datadir=datadir,
+                               timerange=timerange, exchange=exchange, data_handler=data_handler)
 
 
 def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange],
@@ -153,13 +154,14 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
     return data, start_ms
 
 
-def _download_pair_history(datadir: Path,
+def _download_pair_history(pair: str, *,
+                           datadir: Path,
                            exchange: Exchange,
-                           pair: str, *,
-                           new_pairs_days: int = 30,
                            timeframe: str = '5m',
-                           timerange: Optional[TimeRange] = None,
-                           data_handler: IDataHandler = None) -> bool:
+                           process: str = '',
+                           new_pairs_days: int = 30,
+                           data_handler: IDataHandler = None,
+                           timerange: Optional[TimeRange] = None) -> bool:
     """
     Download latest candles from the exchange for the pair and timeframe passed in parameters
     The data is downloaded starting from the last correct data that
@@ -177,7 +179,7 @@ def _download_pair_history(datadir: Path,
 
     try:
         logger.info(
-            f'Download history data for pair: "{pair}", timeframe: {timeframe} '
+            f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe} '
             f'and store in {datadir}.'
         )
 
@@ -194,8 +196,8 @@ def _download_pair_history(datadir: Path,
         new_data = exchange.get_historic_ohlcv(pair=pair,
                                                timeframe=timeframe,
                                                since_ms=since_ms if since_ms else
-                                               int(arrow.utcnow().shift(
-                                                   days=-new_pairs_days).float_timestamp) * 1000
+                                               arrow.utcnow().shift(
+                                                   days=-new_pairs_days).int_timestamp * 1000
                                                )
         # TODO: Maybe move parsing to exchange class (?)
         new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
@@ -234,7 +236,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
     """
     pairs_not_available = []
     data_handler = get_datahandler(datadir, data_format)
-    for pair in pairs:
+    for idx, pair in enumerate(pairs, start=1):
         if pair not in exchange.markets:
             pairs_not_available.append(pair)
             logger.info(f"Skipping pair {pair}...")
@@ -247,10 +249,11 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
                         f'Deleting existing data for pair {pair}, interval {timeframe}.')
 
             logger.info(f'Downloading pair {pair}, interval {timeframe}.')
-            _download_pair_history(datadir=datadir, exchange=exchange,
-                                   pair=pair, timeframe=str(timeframe),
-                                   new_pairs_days=new_pairs_days,
-                                   timerange=timerange, data_handler=data_handler)
+            process = f'{idx}/{len(pairs)}'
+            _download_pair_history(pair=pair, process=process,
+                                   datadir=datadir, exchange=exchange,
+                                   timerange=timerange, data_handler=data_handler,
+                                   timeframe=str(timeframe), new_pairs_days=new_pairs_days)
     return pairs_not_available
 
 
@@ -272,7 +275,7 @@ def _download_trades_history(exchange: Exchange,
             if timerange.stoptype == 'date':
                 until = timerange.stopts * 1000
         else:
-            since = int(arrow.utcnow().shift(days=-new_pairs_days).float_timestamp) * 1000
+            since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000
 
         trades = data_handler.trades_load(pair)
 

freqtrade/data/history/jsondatahandler.py

@@ -62,7 +62,7 @@ class JsonDataHandler(IDataHandler):
         filename = self._pair_data_filename(self._datadir, pair, timeframe)
         _data = data.copy()
         # Convert date to int
-        _data['date'] = _data['date'].astype(np.int64) // 1000 // 1000
+        _data['date'] = _data['date'].view(np.int64) // 1000 // 1000
 
         # Reset index, select only appropriate columns and save as json
         _data.reset_index(drop=True).loc[:, self._columns].to_json(

freqtrade/edge/edge_positioning.py

@@ -151,7 +151,7 @@ class Edge:
         # Fake run-mode to Edge
         prior_rm = self.config['runmode']
         self.config['runmode'] = RunMode.EDGE
-        preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
+        preprocessed = self.strategy.advise_all_indicators(data)
         self.config['runmode'] = prior_rm
 
         # Print timeframe
@@ -231,12 +231,12 @@ class Edge:
                     'Minimum expectancy and minimum winrate are met only for %s,'
                     ' so other pairs are filtered out.',
                     self._final_pairs
-                    )
+                )
             else:
                 logger.info(
                     'Edge removed all pairs as no pair with minimum expectancy '
                     'and minimum winrate was found !'
-                    )
+                )
 
         return self._final_pairs
 
@@ -247,7 +247,7 @@ class Edge:
         final = []
         for pair, info in self._cached_pairs.items():
             if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
-                 info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
+                    info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
                 final.append({
                     'Pair': pair,
                     'Winrate': info.winrate,

freqtrade/enums/__init__.py

@@ -1,6 +1,7 @@
 # flake8: noqa: F401
+from freqtrade.enums.backteststate import BacktestState
 from freqtrade.enums.rpcmessagetype import RPCMessageType
 from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode
 from freqtrade.enums.selltype import SellType
-from freqtrade.enums.signaltype import SignalType
+from freqtrade.enums.signaltype import SignalTagType, SignalType
 from freqtrade.enums.state import State

freqtrade/enums/backteststate.py

@@ -0,0 +1,15 @@
+from enum import Enum
+
+
+class BacktestState(Enum):
+    """
+    Bot application states
+    """
+    STARTUP = 1
+    DATALOAD = 2
+    ANALYZE = 3
+    CONVERT = 4
+    BACKTEST = 5
+
+    def __str__(self):
+        return f"{self.name.lower()}"

freqtrade/enums/runmode.py

@@ -14,6 +14,7 @@ class RunMode(Enum):
     UTIL_EXCHANGE = "util_exchange"
     UTIL_NO_EXCHANGE = "util_no_exchange"
     PLOT = "plot"
+    WEBSERVER = "webserver"
     OTHER = "other"
 
 

freqtrade/enums/signaltype.py

@@ -7,3 +7,10 @@ class SignalType(Enum):
     """
     BUY = "buy"
     SELL = "sell"
+
+
+class SignalTagType(Enum):
+    """
+    Enum for signal columns
+    """
+    BUY_TAG = "buy_tag"

freqtrade/exchange/__init__.py

@@ -15,6 +15,7 @@ from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges,
                                          timeframe_to_seconds, validate_exchange,
                                          validate_exchanges)
 from freqtrade.exchange.ftx import Ftx
+from freqtrade.exchange.gateio import Gateio
 from freqtrade.exchange.hitbtc import Hitbtc
 from freqtrade.exchange.kraken import Kraken
 from freqtrade.exchange.kucoin import Kucoin

freqtrade/exchange/exchange.py

@@ -19,7 +19,8 @@ from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRU
                                             decimal_to_precision)
 from pandas import DataFrame
 
-from freqtrade.constants import DEFAULT_AMOUNT_RESERVE_PERCENT, ListPairsWithTimeframes
+from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES,
+                                 ListPairsWithTimeframes)
 from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
 from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
                                   InvalidOrderException, OperationalException, PricingError,
@@ -387,7 +388,7 @@ class Exchange:
                 # its contents depend on the exchange.
                 # It can also be a string or similar ... so we need to verify that first.
             elif (isinstance(self.markets[pair].get('info', None), dict)
-                  and self.markets[pair].get('info', {}).get('IsRestricted', False)):
+                  and self.markets[pair].get('info', {}).get('prohibitedIn', False)):
                 # Warn users about restricted pairs in whitelist.
                 # We cannot determine reliably if Users are affected.
                 logger.warning(f"Pair {pair} is restricted for some users on this exchange."
@@ -551,7 +552,7 @@ class Exchange:
         amount_reserve_percent = 1.0 + self._config.get('amount_reserve_percent',
                                                         DEFAULT_AMOUNT_RESERVE_PERCENT)
         amount_reserve_percent = (
-          amount_reserve_percent / (1 - abs(stoploss)) if abs(stoploss) != 1 else 1.5
+            amount_reserve_percent / (1 - abs(stoploss)) if abs(stoploss) != 1 else 1.5
         )
         # it should not be more than 50%
         amount_reserve_percent = max(min(amount_reserve_percent, 1.5), 1)
@@ -578,7 +579,7 @@ class Exchange:
             'side': side,
             'remaining': _amount,
             'datetime': arrow.utcnow().isoformat(),
-            'timestamp': int(arrow.utcnow().int_timestamp * 1000),
+            'timestamp': arrow.utcnow().int_timestamp * 1000,
             'status': "closed" if ordertype == "market" else "open",
             'fee': None,
             'info': {}
@@ -618,6 +619,8 @@ class Exchange:
         if self.exchange_has('fetchL2OrderBook'):
             ob = self.fetch_l2_order_book(pair, 20)
             ob_type = 'asks' if side == 'buy' else 'bids'
+            slippage = 0.05
+            max_slippage_val = rate * ((1 + slippage) if side == 'buy' else (1 - slippage))
 
             remaining_amount = amount
             filled_amount = 0
@@ -626,7 +629,9 @@ class Exchange:
                 book_entry_coin_volume = book_entry[1]
                 if remaining_amount > 0:
                     if remaining_amount < book_entry_coin_volume:
+                        # Orderbook at this slot bigger than remaining amount
                         filled_amount += remaining_amount * book_entry_price
+                        break
                     else:
                         filled_amount += book_entry_coin_volume * book_entry_price
                     remaining_amount -= book_entry_coin_volume
@@ -635,7 +640,14 @@ class Exchange:
             else:
                 # If remaining_amount wasn't consumed completely (break was not called)
                 filled_amount += remaining_amount * book_entry_price
-            forecast_avg_filled_price = filled_amount / amount
+            forecast_avg_filled_price = max(filled_amount, 0) / amount
+            # Limit max. slippage to specified value
+            if side == 'buy':
+                forecast_avg_filled_price = min(forecast_avg_filled_price, max_slippage_val)
+
+            else:
+                forecast_avg_filled_price = max(forecast_avg_filled_price, max_slippage_val)
+
             return self.price_to_precision(pair, forecast_avg_filled_price)
 
         return rate
@@ -689,7 +701,16 @@ class Exchange:
     # Order handling
 
     def create_order(self, pair: str, ordertype: str, side: str, amount: float,
-                     rate: float, params: Dict = {}) -> Dict:
+                     rate: float, time_in_force: str = 'gtc') -> Dict:
+
+        if self._config['dry_run']:
+            dry_order = self.create_dry_run_order(pair, ordertype, side, amount, rate)
+            return dry_order
+
+        params = self._params.copy()
+        if time_in_force != 'gtc' and ordertype != 'market':
+            params.update({'timeInForce': time_in_force})
+
         try:
             # Set the precision for amount and price(rate) as accepted by the exchange
             amount = self.amount_to_precision(pair, amount)
@@ -720,32 +741,6 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
-    def buy(self, pair: str, ordertype: str, amount: float,
-            rate: float, time_in_force: str) -> Dict:
-
-        if self._config['dry_run']:
-            dry_order = self.create_dry_run_order(pair, ordertype, "buy", amount, rate)
-            return dry_order
-
-        params = self._params.copy()
-        if time_in_force != 'gtc' and ordertype != 'market':
-            params.update({'timeInForce': time_in_force})
-
-        return self.create_order(pair, ordertype, 'buy', amount, rate, params)
-
-    def sell(self, pair: str, ordertype: str, amount: float,
-             rate: float, time_in_force: str = 'gtc') -> Dict:
-
-        if self._config['dry_run']:
-            dry_order = self.create_dry_run_order(pair, ordertype, "sell", amount, rate)
-            return dry_order
-
-        params = self._params.copy()
-        if time_in_force != 'gtc' and ordertype != 'market':
-            params.update({'timeInForce': time_in_force})
-
-        return self.create_order(pair, ordertype, 'sell', amount, rate, params)
-
     def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool:
         """
         Verify stop_loss against stoploss-order value (limit or price)
@@ -810,7 +805,7 @@ class Exchange:
         :param order: Order dict as returned from fetch_order()
         :return: True if order has been cancelled without being filled, False otherwise.
         """
-        return (order.get('status') in ('closed', 'canceled', 'cancelled')
+        return (order.get('status') in NON_OPEN_EXCHANGE_STATES
                 and order.get('filled') == 0.0)
 
     @retrier
@@ -999,99 +994,64 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
-    def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1,
-                        order_book_min: int = 1):
+    def get_rate(self, pair: str, refresh: bool, side: str) -> float:
         """
-        Helper generator to query orderbook in loop (used for early sell-order placing)
-        """
-        order_book = self.fetch_l2_order_book(pair, order_book_max)
-        for i in range(order_book_min, order_book_max + 1):
-            yield order_book[side][i - 1][0]
-
-    def get_buy_rate(self, pair: str, refresh: bool) -> float:
-        """
-        Calculates bid target between current ask price and last price
+        Calculates bid/ask target
+        bid rate - between current ask price and last price
+        ask rate - either using ticker bid or first bid based on orderbook
+        or remain static in any other case since it's not updating.
         :param pair: Pair to get rate for
         :param refresh: allow cached data
+        :param side: "buy" or "sell"
         :return: float: Price
         :raises PricingError if orderbook price could not be determined.
         """
+        cache_rate: TTLCache = self._buy_rate_cache if side == "buy" else self._sell_rate_cache
+        [strat_name, name] = ['bid_strategy', 'Buy'] if side == "buy" else ['ask_strategy', 'Sell']
+
         if not refresh:
-            rate = self._buy_rate_cache.get(pair)
+            rate = cache_rate.get(pair)
             # Check if cache has been invalidated
             if rate:
-                logger.debug(f"Using cached buy rate for {pair}.")
+                logger.debug(f"Using cached {side} rate for {pair}.")
                 return rate
 
-        bid_strategy = self._config.get('bid_strategy', {})
-        if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False):
+        conf_strategy = self._config.get(strat_name, {})
 
-            order_book_top = bid_strategy.get('order_book_top', 1)
+        if conf_strategy.get('use_order_book', False) and ('use_order_book' in conf_strategy):
+
+            order_book_top = conf_strategy.get('order_book_top', 1)
             order_book = self.fetch_l2_order_book(pair, order_book_top)
             logger.debug('order_book %s', order_book)
             # top 1 = index 0
             try:
-                rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0]
+                rate = order_book[f"{conf_strategy['price_side']}s"][order_book_top - 1][0]
             except (IndexError, KeyError) as e:
                 logger.warning(
-                    "Buy Price from orderbook could not be determined."
-                    f"Orderbook: {order_book}"
-                 )
+                    f"{name} Price at location {order_book_top} from orderbook could not be "
+                    f"determined. Orderbook: {order_book}"
+                )
                 raise PricingError from e
-            logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side "
-                        f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}")
-            used_rate = rate_from_l2
+            price_side = {conf_strategy['price_side'].capitalize()}
+            logger.debug(f"{name} price from orderbook {price_side}"
+                         f"side - top {order_book_top} order book {side} rate {rate:.8f}")
         else:
-            logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price")
+            logger.debug(f"Using Last {conf_strategy['price_side'].capitalize()} / Last Price")
             ticker = self.fetch_ticker(pair)
-            ticker_rate = ticker[bid_strategy['price_side']]
-            if ticker['last'] and ticker_rate > ticker['last']:
-                balance = bid_strategy['ask_last_balance']
-                ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate)
-            used_rate = ticker_rate
-
-        self._buy_rate_cache[pair] = used_rate
-
-        return used_rate
-
-    def get_sell_rate(self, pair: str, refresh: bool) -> float:
-        """
-        Get sell rate - either using ticker bid or first bid based on orderbook
-        or remain static in any other case since it's not updating.
-        :param pair: Pair to get rate for
-        :param refresh: allow cached data
-        :return: Bid rate
-        :raises PricingError if price could not be determined.
-        """
-        if not refresh:
-            rate = self._sell_rate_cache.get(pair)
-            # Check if cache has been invalidated
-            if rate:
-                logger.debug(f"Using cached sell rate for {pair}.")
-                return rate
-
-        ask_strategy = self._config.get('ask_strategy', {})
-        if ask_strategy.get('use_order_book', False):
-            # This code is only used for notifications, selling uses the generator directly
-            logger.info(
-                f"Getting price from order book {ask_strategy['price_side'].capitalize()} side."
-            )
-            try:
-                rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s"))
-            except (IndexError, KeyError) as e:
-                logger.warning("Sell Price at location from orderbook could not be determined.")
-                raise PricingError from e
-        else:
-            ticker = self.fetch_ticker(pair)
-            ticker_rate = ticker[ask_strategy['price_side']]
-            if ticker['last'] and ticker_rate < ticker['last']:
-                balance = ask_strategy.get('bid_last_balance', 0.0)
-                ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last'])
+            ticker_rate = ticker[conf_strategy['price_side']]
+            if ticker['last'] and ticker_rate:
+                if side == 'buy' and ticker_rate > ticker['last']:
+                    balance = conf_strategy['ask_last_balance']
+                    ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate)
+                elif side == 'sell' and ticker_rate < ticker['last']:
+                    balance = conf_strategy.get('bid_last_balance', 0.0)
+                    ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last'])
             rate = ticker_rate
 
         if rate is None:
-            raise PricingError(f"Sell-Rate for {pair} was empty.")
-        self._sell_rate_cache[pair] = rate
+            raise PricingError(f"{name}-Rate for {pair} was empty.")
+        cache_rate[pair] = rate
+
         return rate
 
     # Fee handling
@@ -1294,7 +1254,7 @@ class Exchange:
         logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list))
 
         input_coroutines = []
-
+        cached_pairs = []
         # Gather coroutines to run
         for pair, timeframe in set(pair_list):
             if (((pair, timeframe) not in self._klines)
@@ -1306,6 +1266,7 @@ class Exchange:
                     "Using cached candle (OHLCV) data for pair %s, timeframe %s ...",
                     pair, timeframe
                 )
+                cached_pairs.append((pair, timeframe))
 
         results = asyncio.get_event_loop().run_until_complete(
             asyncio.gather(*input_coroutines, return_exceptions=True))
@@ -1323,11 +1284,15 @@ class Exchange:
                 self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000
             # keeping parsed dataframe in cache
             ohlcv_df = ohlcv_to_dataframe(
-                    ticks, timeframe, pair=pair, fill_missing=True,
-                    drop_incomplete=self._ohlcv_partial_candle)
+                ticks, timeframe, pair=pair, fill_missing=True,
+                drop_incomplete=self._ohlcv_partial_candle)
             results_df[(pair, timeframe)] = ohlcv_df
             if cache:
                 self._klines[(pair, timeframe)] = ohlcv_df
+        # Return cached klines
+        for pair, timeframe in cached_pairs:
+            results_df[(pair, timeframe)] = self.klines((pair, timeframe), copy=False)
+
         return results_df
 
     def _now_is_time_to_refresh(self, pair: str, timeframe: str) -> bool:
@@ -1538,7 +1503,7 @@ class Exchange:
         :returns List of trade data
         """
         if not self.exchange_has("fetchTrades"):
-            raise OperationalException("This exchange does not suport downloading Trades.")
+            raise OperationalException("This exchange does not support downloading Trades.")
 
         return asyncio.get_event_loop().run_until_complete(
             self._async_get_trade_history(pair=pair, since=since,

freqtrade/exchange/gateio.py

@@ -0,0 +1,23 @@
+""" Gate.io exchange subclass """
+import logging
+from typing import Dict
+
+from freqtrade.exchange import Exchange
+
+
+logger = logging.getLogger(__name__)
+
+
+class Gateio(Exchange):
+    """
+    Gate.io exchange class. Contains adjustments needed for Freqtrade to work
+    with this exchange.
+
+    Please note that this exchange is not included in the list of exchanges
+    officially supported by the Freqtrade development team. So some features
+    may still not work as expected.
+    """
+
+    _ft_has: Dict = {
+        "ohlcv_candle_limit": 1000,
+    }

freqtrade/freqtradebot.py

@@ -420,26 +420,24 @@ class FreqtradeBot(LoggingMixin):
             return False
 
         # running get_signal on historical data fetched
-        (buy, sell) = self.strategy.get_signal(pair, self.strategy.timeframe, analyzed_df)
+        (buy, sell, buy_tag) = self.strategy.get_signal(
+            pair,
+            self.strategy.timeframe,
+            analyzed_df
+        )
 
         if buy and not sell:
             stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge)
-            if not stake_amount:
-                logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.")
-                return False
-
-            logger.info(f"Buy signal found: about create a new trade for {pair} with stake_amount: "
-                        f"{stake_amount} ...")
 
             bid_check_dom = self.config.get('bid_strategy', {}).get('check_depth_of_market', {})
             if ((bid_check_dom.get('enabled', False)) and
                     (bid_check_dom.get('bids_to_ask_delta', 0) > 0)):
                 if self._check_depth_of_market_buy(pair, bid_check_dom):
-                    return self.execute_buy(pair, stake_amount)
+                    return self.execute_entry(pair, stake_amount, buy_tag=buy_tag)
                 else:
                     return False
 
-            return self.execute_buy(pair, stake_amount)
+            return self.execute_entry(pair, stake_amount, buy_tag=buy_tag)
         else:
             return False
 
@@ -467,8 +465,8 @@ class FreqtradeBot(LoggingMixin):
             logger.info(f"Bids to asks delta for {pair} does not satisfy condition.")
             return False
 
-    def execute_buy(self, pair: str, stake_amount: float, price: Optional[float] = None,
-                    forcebuy: bool = False) -> bool:
+    def execute_entry(self, pair: str, stake_amount: float, price: Optional[float] = None,
+                      forcebuy: bool = False, buy_tag: Optional[str] = None) -> bool:
         """
         Executes a limit buy for the given pair
         :param pair: pair for which we want to create a LIMIT_BUY
@@ -481,20 +479,35 @@ class FreqtradeBot(LoggingMixin):
             buy_limit_requested = price
         else:
             # Calculate price
-            buy_limit_requested = self.exchange.get_buy_rate(pair, True)
+            proposed_buy_rate = self.exchange.get_rate(pair, refresh=True, side="buy")
+            custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price,
+                                                       default_retval=proposed_buy_rate)(
+                pair=pair, current_time=datetime.now(timezone.utc),
+                proposed_rate=proposed_buy_rate)
+
+            buy_limit_requested = self.get_valid_price(custom_entry_price, proposed_buy_rate)
 
         if not buy_limit_requested:
             raise PricingError('Could not determine buy price.')
 
         min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, buy_limit_requested,
                                                                    self.strategy.stoploss)
-        if min_stake_amount is not None and min_stake_amount > stake_amount:
-            logger.warning(
-                f"Can't open a new trade for {pair}: stake amount "
-                f"is too small ({stake_amount} < {min_stake_amount})"
-            )
+
+        if not self.edge:
+            max_stake_amount = self.wallets.get_available_stake_amount()
+            stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount,
+                                                 default_retval=stake_amount)(
+                pair=pair, current_time=datetime.now(timezone.utc),
+                current_rate=buy_limit_requested, proposed_stake=stake_amount,
+                min_stake=min_stake_amount, max_stake=max_stake_amount)
+        stake_amount = self.wallets._validate_stake_amount(pair, stake_amount, min_stake_amount)
+
+        if not stake_amount:
             return False
 
+        logger.info(f"Buy signal found: about create a new trade for {pair} with stake_amount: "
+                    f"{stake_amount} ...")
+
         amount = stake_amount / buy_limit_requested
         order_type = self.strategy.order_types['buy']
         if forcebuy:
@@ -507,9 +520,9 @@ class FreqtradeBot(LoggingMixin):
             logger.info(f"User requested abortion of buying {pair}")
             return False
         amount = self.exchange.amount_to_precision(pair, amount)
-        order = self.exchange.buy(pair=pair, ordertype=order_type,
-                                  amount=amount, rate=buy_limit_requested,
-                                  time_in_force=time_in_force)
+        order = self.exchange.create_order(pair=pair, ordertype=order_type, side="buy",
+                                           amount=amount, rate=buy_limit_requested,
+                                           time_in_force=time_in_force)
         order_obj = Order.parse_from_ccxt_object(order, pair, 'buy')
         order_id = order['id']
         order_status = order.get('status', None)
@@ -562,6 +575,7 @@ class FreqtradeBot(LoggingMixin):
             exchange=self.exchange.id,
             open_order_id=order_id,
             strategy=self.strategy.get_strategy_name(),
+            buy_tag=buy_tag,
             timeframe=timeframe_to_minutes(self.config['timeframe'])
         )
         trade.orders.append(order_obj)
@@ -587,6 +601,7 @@ class FreqtradeBot(LoggingMixin):
         msg = {
             'trade_id': trade.id,
             'type': RPCMessageType.BUY,
+            'buy_tag': trade.buy_tag,
             'exchange': self.exchange.name.capitalize(),
             'pair': trade.pair,
             'limit': trade.open_rate,
@@ -606,11 +621,12 @@ class FreqtradeBot(LoggingMixin):
         """
         Sends rpc notification when a buy cancel occurred.
         """
-        current_rate = self.exchange.get_buy_rate(trade.pair, False)
+        current_rate = self.exchange.get_rate(trade.pair, refresh=False, side="buy")
 
         msg = {
             'trade_id': trade.id,
             'type': RPCMessageType.BUY_CANCEL,
+            'buy_tag': trade.buy_tag,
             'exchange': self.exchange.name.capitalize(),
             'pair': trade.pair,
             'limit': trade.open_rate,
@@ -631,6 +647,7 @@ class FreqtradeBot(LoggingMixin):
         msg = {
             'trade_id': trade.id,
             'type': RPCMessageType.BUY_FILL,
+            'buy_tag': trade.buy_tag,
             'exchange': self.exchange.name.capitalize(),
             'pair': trade.pair,
             'open_rate': trade.open_rate,
@@ -684,46 +701,21 @@ class FreqtradeBot(LoggingMixin):
 
         (buy, sell) = (False, False)
 
-        config_ask_strategy = self.config.get('ask_strategy', {})
-
-        if (config_ask_strategy.get('use_sell_signal', True) or
-                config_ask_strategy.get('ignore_roi_if_buy_signal', False)):
+        if (self.config.get('use_sell_signal', True) or
+                self.config.get('ignore_roi_if_buy_signal', False)):
             analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair,
                                                                       self.strategy.timeframe)
 
-            (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.timeframe, analyzed_df)
+            (buy, sell, _) = self.strategy.get_signal(
+                trade.pair,
+                self.strategy.timeframe,
+                analyzed_df
+            )
 
-        if config_ask_strategy.get('use_order_book', False):
-            order_book_min = config_ask_strategy.get('order_book_min', 1)
-            order_book_max = config_ask_strategy.get('order_book_max', 1)
-            logger.debug(f'Using order book between {order_book_min} and {order_book_max} '
-                         f'for selling {trade.pair}...')
-
-            order_book = self.exchange._order_book_gen(
-                trade.pair, f"{config_ask_strategy['price_side']}s",
-                order_book_min=order_book_min, order_book_max=order_book_max)
-            for i in range(order_book_min, order_book_max + 1):
-                try:
-                    sell_rate = next(order_book)
-                except (IndexError, KeyError) as e:
-                    logger.warning(
-                        f"Sell Price at location {i} from orderbook could not be determined."
-                    )
-                    raise PricingError from e
-                logger.debug(f"  order book {config_ask_strategy['price_side']} top {i}: "
-                             f"{sell_rate:0.8f}")
-                # Assign sell-rate to cache - otherwise sell-rate is never updated in the cache,
-                # resulting in outdated RPC messages
-                self.exchange._sell_rate_cache[trade.pair] = sell_rate
-
-                if self._check_and_execute_sell(trade, sell_rate, buy, sell):
-                    return True
-
-        else:
-            logger.debug('checking sell')
-            sell_rate = self.exchange.get_sell_rate(trade.pair, True)
-            if self._check_and_execute_sell(trade, sell_rate, buy, sell):
-                return True
+        logger.debug('checking sell')
+        sell_rate = self.exchange.get_rate(trade.pair, refresh=True, side="sell")
+        if self._check_and_execute_sell(trade, sell_rate, buy, sell):
+            return True
 
         logger.debug('Found no sell signal for %s.', trade)
         return False
@@ -753,7 +745,7 @@ class FreqtradeBot(LoggingMixin):
             trade.stoploss_order_id = None
             logger.error(f'Unable to place a stoploss order on exchange. {e}')
             logger.warning('Selling the trade forcefully')
-            self.execute_sell(trade, trade.stop_loss, sell_reason=SellCheckTuple(
+            self.execute_trade_exit(trade, trade.stop_loss, sell_reason=SellCheckTuple(
                 sell_type=SellType.EMERGENCY_SELL))
 
         except ExchangeError:
@@ -871,7 +863,7 @@ class FreqtradeBot(LoggingMixin):
 
         if should_sell.sell_flag:
             logger.info(f'Executing Sell for {trade.pair}. Reason: {should_sell.sell_type}')
-            self.execute_sell(trade, sell_rate, should_sell)
+            self.execute_trade_exit(trade, sell_rate, should_sell)
             return True
         return False
 
@@ -953,7 +945,7 @@ class FreqtradeBot(LoggingMixin):
         was_trade_fully_canceled = False
 
         # Cancelled orders may have the status of 'canceled' or 'closed'
-        if order['status'] not in ('cancelled', 'canceled', 'closed'):
+        if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
             filled_val = order.get('filled', 0.0) or 0.0
             filled_stake = filled_val * trade.open_rate
             minstake = self.exchange.get_min_pair_stake_amount(
@@ -969,7 +961,7 @@ class FreqtradeBot(LoggingMixin):
             # Avoid race condition where the order could not be cancelled coz its already filled.
             # Simply bailing here is the only safe way - as this order will then be
             # handled in the next iteration.
-            if corder.get('status') not in ('cancelled', 'canceled', 'closed'):
+            if corder.get('status') not in constants.NON_OPEN_EXCHANGE_STATES:
                 logger.warning(f"Order {trade.open_order_id} for {trade.pair} not cancelled.")
                 return False
         else:
@@ -991,7 +983,7 @@ class FreqtradeBot(LoggingMixin):
             # if trade is partially complete, edit the stake details for the trade
             # and close the order
             # cancel_order may not contain the full order dict, so we need to fallback
-            # to the order dict aquired before cancelling.
+            # to the order dict acquired before cancelling.
             # we need to fall back to the values from order if corder does not contain these keys.
             trade.amount = filled_amount
             trade.stake_amount = trade.amount * trade.open_rate
@@ -1072,9 +1064,9 @@ class FreqtradeBot(LoggingMixin):
             raise DependencyException(
                 f"Not enough amount to sell. Trade-amount: {amount}, Wallet: {wallet_amount}")
 
-    def execute_sell(self, trade: Trade, limit: float, sell_reason: SellCheckTuple) -> bool:
+    def execute_trade_exit(self, trade: Trade, limit: float, sell_reason: SellCheckTuple) -> bool:
         """
-        Executes a limit sell for the given trade and limit
+        Executes a trade exit for the given trade and limit
         :param trade: Trade instance
         :param limit: limit rate for the sell order
         :param sell_reason: Reason the sell was triggered
@@ -1090,6 +1082,17 @@ class FreqtradeBot(LoggingMixin):
            and self.strategy.order_types['stoploss_on_exchange']:
             limit = trade.stop_loss
 
+        # set custom_exit_price if available
+        proposed_limit_rate = limit
+        current_profit = trade.calc_profit_ratio(limit)
+        custom_exit_price = strategy_safe_wrapper(self.strategy.custom_exit_price,
+                                                  default_retval=proposed_limit_rate)(
+            pair=trade.pair, trade=trade,
+            current_time=datetime.now(timezone.utc),
+            proposed_rate=proposed_limit_rate, current_profit=current_profit)
+
+        limit = self.get_valid_price(custom_exit_price, proposed_limit_rate)
+
         # First cancelling stoploss on exchange ...
         if self.strategy.order_types.get('stoploss_on_exchange') and trade.stoploss_order_id:
             try:
@@ -1120,11 +1123,11 @@ class FreqtradeBot(LoggingMixin):
 
         try:
             # Execute sell and update trade record
-            order = self.exchange.sell(pair=trade.pair,
-                                       ordertype=order_type,
-                                       amount=amount, rate=limit,
-                                       time_in_force=time_in_force
-                                       )
+            order = self.exchange.create_order(pair=trade.pair,
+                                               ordertype=order_type, side="sell",
+                                               amount=amount, rate=limit,
+                                               time_in_force=time_in_force
+                                               )
         except InsufficientFundsError as e:
             logger.warning(f"Unable to place order {e}.")
             # Try to figure out what went wrong
@@ -1139,7 +1142,7 @@ class FreqtradeBot(LoggingMixin):
         trade.close_rate_requested = limit
         trade.sell_reason = sell_reason.sell_reason
         # In case of market sell orders the order can be closed immediately
-        if order.get('status', 'unknown') == 'closed':
+        if order.get('status', 'unknown') in ('closed', 'expired'):
             self.update_trade_state(trade, trade.open_order_id, order)
         Trade.commit()
 
@@ -1158,7 +1161,8 @@ class FreqtradeBot(LoggingMixin):
         profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
         profit_trade = trade.calc_profit(rate=profit_rate)
         # Use cached rates here - it was updated seconds ago.
-        current_rate = self.exchange.get_sell_rate(trade.pair, False) if not fill else None
+        current_rate = self.exchange.get_rate(
+            trade.pair, refresh=False, side="sell") if not fill else None
         profit_ratio = trade.calc_profit_ratio(profit_rate)
         gain = "profit" if profit_ratio > 0 else "loss"
 
@@ -1203,7 +1207,7 @@ class FreqtradeBot(LoggingMixin):
 
         profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
         profit_trade = trade.calc_profit(rate=profit_rate)
-        current_rate = self.exchange.get_sell_rate(trade.pair, False)
+        current_rate = self.exchange.get_rate(trade.pair, refresh=False, side="sell")
         profit_ratio = trade.calc_profit_ratio(profit_rate)
         gain = "profit" if profit_ratio > 0 else "loss"
 
@@ -1377,7 +1381,9 @@ class FreqtradeBot(LoggingMixin):
         if fee_currency:
             # fee_rate should use mean
             fee_rate = sum(fee_rate_array) / float(len(fee_rate_array)) if fee_rate_array else None
-            trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', ''))
+            if fee_rate is not None and fee_rate < 0.02:
+                # Only update if fee-rate is < 2%
+                trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', ''))
 
         if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC):
             logger.warning(f"Amount {amount} does not match amount {trade.amount}")
@@ -1388,3 +1394,26 @@ class FreqtradeBot(LoggingMixin):
                                               amount=amount, fee_abs=fee_abs)
         else:
             return amount
+
+    def get_valid_price(self, custom_price: float, proposed_price: float) -> float:
+        """
+        Return the valid price.
+        Check if the custom price is of the good type if not return proposed_price
+        :return: valid price for the order
+        """
+        if custom_price:
+            try:
+                valid_custom_price = float(custom_price)
+            except ValueError:
+                valid_custom_price = proposed_price
+        else:
+            valid_custom_price = proposed_price
+
+        cust_p_max_dist_r = self.config.get('custom_price_max_distance_ratio', 0.02)
+        min_custom_price_allowed = proposed_price - (proposed_price * cust_p_max_dist_r)
+        max_custom_price_allowed = proposed_price + (proposed_price * cust_p_max_dist_r)
+
+        # Bracket between min_custom_price_allowed and max_custom_price_allowed
+        return max(
+            min(valid_custom_price, max_custom_price_allowed),
+            min_custom_price_allowed)

freqtrade/main.py

@@ -44,7 +44,7 @@ def main(sysargv: List[str] = None) -> None:
                 "as `freqtrade trade [options...]`.\n"
                 "To see the full list of options available, please use "
                 "`freqtrade --help` or `freqtrade <command> --help`."
-                )
+            )
 
     except SystemExit as e:
         return_code = e

freqtrade/misc.py

@@ -8,6 +8,7 @@ from datetime import datetime
 from pathlib import Path
 from typing import Any, Iterator, List
 from typing.io import IO
+from urllib.parse import urlparse
 
 import rapidjson
 
@@ -214,3 +215,16 @@ def chunks(lst: List[Any], n: int) -> Iterator[List[Any]]:
     """
     for chunk in range(0, len(lst), n):
         yield (lst[chunk:chunk + n])
+
+
+def parse_db_uri_for_logging(uri: str):
+    """
+    Helper method to parse the DB URI and return the same DB URI with the password censored
+    if it contains it. Otherwise, return the DB URI unchanged
+    :param uri: DB URI to parse for logging
+    """
+    parsed_db_uri = urlparse(uri)
+    if not parsed_db_uri.netloc:  # No need for censoring as no password was provided
+        return uri
+    pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0]
+    return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')

freqtrade/optimize/backtesting.py

@@ -15,12 +15,13 @@ from freqtrade.configuration import TimeRange, remove_credentials, validate_conf
 from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data import history
 from freqtrade.data.btanalysis import trade_list_to_dataframe
-from freqtrade.data.converter import trim_dataframes
+from freqtrade.data.converter import trim_dataframe, trim_dataframes
 from freqtrade.data.dataprovider import DataProvider
-from freqtrade.enums import SellType
+from freqtrade.enums import BacktestState, SellType
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
 from freqtrade.mixins import LoggingMixin
+from freqtrade.optimize.bt_progress import BTProgress
 from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
                                                  store_backtest_stats)
 from freqtrade.persistence import LocalTrade, PairLocks, Trade
@@ -42,6 +43,7 @@ CLOSE_IDX = 3
 SELL_IDX = 4
 LOW_IDX = 5
 HIGH_IDX = 6
+BUY_TAG_IDX = 7
 
 
 class Backtesting:
@@ -57,6 +59,7 @@ class Backtesting:
 
         LoggingMixin.show_output = False
         self.config = config
+        self.results: Optional[Dict[str, Any]] = None
 
         # Reset keys for backtesting
         remove_credentials(self.config)
@@ -114,10 +117,22 @@ class Backtesting:
 
         self.wallets = Wallets(self.config, self.exchange, log=False)
 
+        self.timerange = TimeRange.parse_timerange(
+            None if self.config.get('timerange') is None else str(self.config.get('timerange')))
+
         # Get maximum required startup period
         self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
+        # Add maximum startup candle count to configuration for informative pairs support
+        self.config['startup_candle_count'] = self.required_startup
+        self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
+
+        self.progress = BTProgress()
+        self.abort = False
 
     def __del__(self):
+        self.cleanup()
+
+    def cleanup(self):
         LoggingMixin.show_output = True
         PairLocks.use_db = True
         Trade.use_db = True
@@ -128,10 +143,14 @@ class Backtesting:
         """
         self.strategy: IStrategy = strategy
         strategy.dp = self.dataprovider
+        # Attach Wallets to Strategy baseclass
+        IStrategy.wallets = self.wallets
         # Set stoploss_on_exchange to false for backtesting,
         # since a "perfect" stoploss-sell is assumed anyway
         # And the regular "stoploss" function would not apply to that case
         self.strategy.order_types['stoploss_on_exchange'] = False
+
+    def _load_protections(self, strategy: IStrategy):
         if self.config.get('enable_protections', False):
             conf = self.config
             if hasattr(strategy, 'protections'):
@@ -144,14 +163,13 @@ class Backtesting:
         Loads backtest data and returns the data combined with the timerange
         as tuple.
         """
-        timerange = TimeRange.parse_timerange(None if self.config.get(
-            'timerange') is None else str(self.config.get('timerange')))
+        self.progress.init_step(BacktestState.DATALOAD, 1)
 
         data = history.load_data(
             datadir=self.config['datadir'],
             pairs=self.pairlists.whitelist,
             timeframe=self.timeframe,
-            timerange=timerange,
+            timerange=self.timerange,
             startup_candles=self.required_startup,
             fail_without_data=True,
             data_format=self.config.get('dataformat_ohlcv', 'json'),
@@ -164,10 +182,11 @@ class Backtesting:
                     f'({(max_date - min_date).days} days).')
 
         # Adjust startts forward if not enough data is available
-        timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
-                                            self.required_startup, min_date)
+        self.timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
+                                                 self.required_startup, min_date)
 
-        return data, timerange
+        self.progress.set_new_value(1)
+        return data, self.timerange
 
     def prepare_backtest(self, enable_protections):
         """
@@ -180,6 +199,16 @@ class Backtesting:
         Trade.reset_trades()
         self.rejected_trades = 0
         self.dataprovider.clear_cache()
+        self._load_protections(self.strategy)
+
+    def check_abort(self):
+        """
+        Check if abort was requested, raise DependencyException if that's the case
+        Only applies to Interactive backtest mode (webserver mode)
+        """
+        if self.abort:
+            self.abort = False
+            raise DependencyException("Stop requested")
 
     def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]:
         """
@@ -189,27 +218,38 @@ class Backtesting:
         """
         # Every change to this headers list must evaluate further usages of the resulting tuple
         # and eventually change the constants for indexes at the top
-        headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
+        headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high', 'buy_tag']
         data: Dict = {}
+        self.progress.init_step(BacktestState.CONVERT, len(processed))
+
         # Create dict with data
         for pair, pair_data in processed.items():
+            self.check_abort()
+            self.progress.increment()
             if not pair_data.empty:
                 pair_data.loc[:, 'buy'] = 0  # cleanup if buy_signal is exist
                 pair_data.loc[:, 'sell'] = 0  # cleanup if sell_signal is exist
+                pair_data.loc[:, 'buy_tag'] = None  # cleanup if buy_tag is exist
 
             df_analyzed = self.strategy.advise_sell(
-                self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
-
+                self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair}).copy()
+            # Trim startup period from analyzed dataframe
+            df_analyzed = trim_dataframe(df_analyzed, self.timerange,
+                                         startup_candles=self.required_startup)
             # To avoid using data from future, we use buy/sell signals shifted
             # from the previous candle
             df_analyzed.loc[:, 'buy'] = df_analyzed.loc[:, 'buy'].shift(1)
             df_analyzed.loc[:, 'sell'] = df_analyzed.loc[:, 'sell'].shift(1)
+            df_analyzed.loc[:, 'buy_tag'] = df_analyzed.loc[:, 'buy_tag'].shift(1)
 
-            df_analyzed.drop(df_analyzed.head(1).index, inplace=True)
+            # Update dataprovider cache
+            self.dataprovider._set_cached_df(pair, self.timeframe, df_analyzed)
+
+            df_analyzed = df_analyzed.drop(df_analyzed.head(1).index)
 
             # Convert from Pandas to list for performance reasons
             # (Looping Pandas is slow.)
-            data[pair] = df_analyzed.values.tolist()
+            data[pair] = df_analyzed[headers].values.tolist()
         return data
 
     def _get_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple,
@@ -228,16 +268,20 @@ class Backtesting:
             # Special case: trailing triggers within same candle as trade opened. Assume most
             # pessimistic price movement, which is moving just enough to arm stoploss and
             # immediately going down to stop price.
-            if (sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0
-                    and self.strategy.trailing_stop_positive):
-                if self.strategy.trailing_only_offset_is_reached:
+            if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0:
+                if (
+                    not self.strategy.use_custom_stoploss and self.strategy.trailing_stop
+                    and self.strategy.trailing_only_offset_is_reached
+                    and self.strategy.trailing_stop_positive_offset is not None
+                    and self.strategy.trailing_stop_positive
+                ):
                     # Worst case: price reaches stop_positive_offset and dives down.
                     stop_rate = (sell_row[OPEN_IDX] *
                                  (1 + abs(self.strategy.trailing_stop_positive_offset) -
-                                 abs(self.strategy.trailing_stop_positive)))
+                                  abs(self.strategy.trailing_stop_positive)))
                 else:
                     # Worst case: price ticks tiny bit above open and dives down.
-                    stop_rate = sell_row[OPEN_IDX] * (1 - abs(self.strategy.trailing_stop_positive))
+                    stop_rate = sell_row[OPEN_IDX] * (1 - abs(trade.stop_loss_pct))
                     assert stop_rate < sell_row[HIGH_IDX]
                 return stop_rate
 
@@ -275,14 +319,14 @@ class Backtesting:
             return sell_row[OPEN_IDX]
 
     def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]:
-
+        sell_candle_time = sell_row[DATE_IDX].to_pydatetime()
         sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX],  # type: ignore
-                                         sell_row[DATE_IDX].to_pydatetime(), sell_row[BUY_IDX],
+                                         sell_candle_time, sell_row[BUY_IDX],
                                          sell_row[SELL_IDX],
                                          low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX])
 
         if sell.sell_flag:
-            trade.close_date = sell_row[DATE_IDX].to_pydatetime()
+            trade.close_date = sell_candle_time
             trade.sell_reason = sell.sell_reason
             trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60)
             closerate = self._get_close_rate(sell_row, trade, sell, trade_dur)
@@ -294,7 +338,7 @@ class Backtesting:
                     rate=closerate,
                     time_in_force=time_in_force,
                     sell_reason=sell.sell_reason,
-                    current_time=sell_row[DATE_IDX].to_pydatetime()):
+                    current_time=sell_candle_time):
                 return None
 
             trade.close(closerate, show_msg=False)
@@ -307,7 +351,18 @@ class Backtesting:
             stake_amount = self.wallets.get_trade_stake_amount(pair, None)
         except DependencyException:
             return None
-        min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05)
+
+        min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05) or 0
+        max_stake_amount = self.wallets.get_available_stake_amount()
+
+        stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount,
+                                             default_retval=stake_amount)(
+            pair=pair, current_time=row[DATE_IDX].to_pydatetime(), current_rate=row[OPEN_IDX],
+            proposed_stake=stake_amount, min_stake=min_stake_amount, max_stake=max_stake_amount)
+        stake_amount = self.wallets._validate_stake_amount(pair, stake_amount, min_stake_amount)
+
+        if not stake_amount:
+            return None
 
         order_type = self.strategy.order_types['buy']
         time_in_force = self.strategy.order_time_in_force['sell']
@@ -319,6 +374,7 @@ class Backtesting:
 
         if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount):
             # Enter trade
+            has_buy_tag = len(row) >= BUY_TAG_IDX + 1
             trade = LocalTrade(
                 pair=pair,
                 open_rate=row[OPEN_IDX],
@@ -328,6 +384,7 @@ class Backtesting:
                 fee_open=self.fee,
                 fee_close=self.fee,
                 is_open=True,
+                buy_tag=row[BUY_TAG_IDX] if has_buy_tag else None,
                 exchange='backtesting',
             )
             return trade
@@ -384,10 +441,6 @@ class Backtesting:
         trades: List[LocalTrade] = []
         self.prepare_backtest(enable_protections)
 
-        # Update dataprovider cache
-        for pair, dataframe in processed.items():
-            self.dataprovider._set_cached_df(pair, self.timeframe, dataframe)
-
         # Use dict of lists with data for performance
         # (looping lists is a lot faster than pandas DataFrames)
         data: Dict = self._get_ohlcv_as_lists(processed)
@@ -399,13 +452,18 @@ class Backtesting:
         open_trades: Dict[str, List[LocalTrade]] = defaultdict(list)
         open_trade_count = 0
 
+        self.progress.init_step(BacktestState.BACKTEST, int(
+            (end_date - start_date) / timedelta(minutes=self.timeframe_min)))
+
         # Loop timerange and get candle for each pair at that point in time
         while tmp <= end_date:
             open_trade_count_start = open_trade_count
-
+            self.check_abort()
             for i, pair in enumerate(data):
                 row_index = indexes[pair]
                 try:
+                    # Row is treated as "current incomplete candle".
+                    # Buy / sell signals are shifted by 1 to compensate for this.
                     row = data[pair][row_index]
                 except IndexError:
                     # missing Data for one pair at the end.
@@ -417,8 +475,8 @@ class Backtesting:
                     continue
 
                 row_index += 1
-                self.dataprovider._set_dataframe_max_index(row_index)
                 indexes[pair] = row_index
+                self.dataprovider._set_dataframe_max_index(row_index)
 
                 # without positionstacking, we can only have one open trade per pair.
                 # max_open_trades must be respected
@@ -442,7 +500,7 @@ class Backtesting:
                         open_trades[pair].append(trade)
                         LocalTrade.add_bt_trade(trade)
 
-                for trade in open_trades[pair]:
+                for trade in list(open_trades[pair]):
                     # also check the buying candle for sell conditions.
                     trade_entry = self._get_sell_trade_entry(trade, row)
                     # Sell occurred
@@ -458,6 +516,7 @@ class Backtesting:
                             self.protections.global_stop(tmp)
 
             # Move time one configured time_interval ahead.
+            self.progress.increment()
             tmp += timedelta(minutes=self.timeframe_min)
 
         trades += self.handle_left_open(open_trades, data=data)
@@ -472,7 +531,10 @@ class Backtesting:
             'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']),
         }
 
-    def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange):
+    def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, DataFrame],
+                              timerange: TimeRange):
+        self.progress.init_step(BacktestState.ANALYZE, 0)
+
         logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
         backtest_start_time = datetime.now(timezone.utc)
         self._set_strategy(strat)
@@ -489,16 +551,18 @@ class Backtesting:
             max_open_trades = 0
 
         # need to reprocess data every time to populate signals
-        preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
+        preprocessed = self.strategy.advise_all_indicators(data)
 
         # Trim startup period from analyzed dataframe
-        preprocessed = trim_dataframes(preprocessed, timerange, self.required_startup)
+        preprocessed_tmp = trim_dataframes(preprocessed, timerange, self.required_startup)
 
-        if not preprocessed:
+        if not preprocessed_tmp:
             raise OperationalException(
                 "No data left after adjusting for startup candles.")
 
-        min_date, max_date = history.get_timerange(preprocessed)
+        # Use preprocessed_tmp for date generation (the trimmed dataframe).
+        # Backtesting will re-trim the dataframes after buy/sell signal generation.
+        min_date, max_date = history.get_timerange(preprocessed_tmp)
         logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'({(max_date - min_date).days} days).')
@@ -533,11 +597,12 @@ class Backtesting:
         for strat in self.strategylist:
             min_date, max_date = self.backtest_one_strategy(strat, data, timerange)
         if len(self.strategylist) > 0:
-            stats = generate_backtest_stats(data, self.all_results,
-                                            min_date=min_date, max_date=max_date)
+
+            self.results = generate_backtest_stats(data, self.all_results,
+                                                   min_date=min_date, max_date=max_date)
 
             if self.config.get('export', 'none') == 'trades':
-                store_backtest_stats(self.config['exportfilename'], stats)
+                store_backtest_stats(self.config['exportfilename'], self.results)
 
             # Show backtest results
-            show_backtest_results(self.config, stats)
+            show_backtest_results(self.config, self.results)

freqtrade/optimize/bt_progress.py

@@ -0,0 +1,33 @@
+from freqtrade.enums import BacktestState
+
+
+class BTProgress:
+    _action: BacktestState = BacktestState.STARTUP
+    _progress: float = 0
+    _max_steps: float = 0
+
+    def __init__(self):
+        pass
+
+    def init_step(self, action: BacktestState, max_steps: float):
+        self._action = action
+        self._max_steps = max_steps
+        self._proress = 0
+
+    def set_new_value(self, new_value: float):
+        self._progress = new_value
+
+    def increment(self):
+        self._progress += 1
+
+    @property
+    def progress(self):
+        """
+        Get progress as ratio, capped to be between 0 and 1 (to avoid small calculation errors).
+        """
+        return max(min(round(self._progress / self._max_steps, 5)
+                       if self._max_steps > 0 else 0, 1), 0)
+
+    @property
+    def action(self):
+        return str(self._action)

freqtrade/optimize/hyperopt.py

@@ -12,7 +12,6 @@ from math import ceil
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
-import numpy as np
 import progressbar
 import rapidjson
 from colorama import Fore, Style
@@ -20,16 +19,16 @@ from colorama import init as colorama_init
 from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects
 from pandas import DataFrame
 
-from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN
+from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN
 from freqtrade.data.converter import trim_dataframes
 from freqtrade.data.history import get_timerange
-from freqtrade.misc import file_dump_json, plural
+from freqtrade.misc import deep_merge_dicts, file_dump_json, plural
 from freqtrade.optimize.backtesting import Backtesting
 # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
 from freqtrade.optimize.hyperopt_auto import HyperOptAuto
 from freqtrade.optimize.hyperopt_interface import IHyperOpt  # noqa: F401
 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss  # noqa: F401
-from freqtrade.optimize.hyperopt_tools import HyperoptTools
+from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer
 from freqtrade.optimize.optimize_reports import generate_strategy_stats
 from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver
 
@@ -67,6 +66,7 @@ class Hyperopt:
     def __init__(self, config: Dict[str, Any]) -> None:
         self.buy_space: List[Dimension] = []
         self.sell_space: List[Dimension] = []
+        self.protection_space: List[Dimension] = []
         self.roi_space: List[Dimension] = []
         self.stoploss_space: List[Dimension] = []
         self.trailing_space: List[Dimension] = []
@@ -78,8 +78,11 @@ class Hyperopt:
 
         if not self.config.get('hyperopt'):
             self.custom_hyperopt = HyperOptAuto(self.config)
+            self.auto_hyperopt = True
         else:
             self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config)
+            self.auto_hyperopt = False
+
         self.backtesting._set_strategy(self.backtesting.strategylist[0])
         self.custom_hyperopt.strategy = self.backtesting.strategy
 
@@ -100,16 +103,30 @@ class Hyperopt:
         self.num_epochs_saved = 0
         self.current_best_epoch: Optional[Dict[str, Any]] = None
 
-        # Populate functions here (hasattr is slow so should not be run during "regular" operations)
-        if hasattr(self.custom_hyperopt, 'populate_indicators'):
-            self.backtesting.strategy.advise_indicators = (  # type: ignore
-                self.custom_hyperopt.populate_indicators)  # type: ignore
-        if hasattr(self.custom_hyperopt, 'populate_buy_trend'):
-            self.backtesting.strategy.advise_buy = (  # type: ignore
-                self.custom_hyperopt.populate_buy_trend)  # type: ignore
-        if hasattr(self.custom_hyperopt, 'populate_sell_trend'):
-            self.backtesting.strategy.advise_sell = (  # type: ignore
-                self.custom_hyperopt.populate_sell_trend)  # type: ignore
+        if not self.auto_hyperopt:
+            # Populate "fallback" functions here
+            # (hasattr is slow so should not be run during "regular" operations)
+            if hasattr(self.custom_hyperopt, 'populate_indicators'):
+                logger.warning(
+                    "DEPRECATED: Using `populate_indicators()` in the hyperopt file is deprecated. "
+                    "Please move these methods to your strategy."
+                    )
+                self.backtesting.strategy.populate_indicators = (  # type: ignore
+                    self.custom_hyperopt.populate_indicators)  # type: ignore
+            if hasattr(self.custom_hyperopt, 'populate_buy_trend'):
+                logger.warning(
+                    "DEPRECATED: Using `populate_buy_trend()` in the hyperopt file is deprecated. "
+                    "Please move these methods to your strategy."
+                )
+                self.backtesting.strategy.populate_buy_trend = (  # type: ignore
+                    self.custom_hyperopt.populate_buy_trend)  # type: ignore
+            if hasattr(self.custom_hyperopt, 'populate_sell_trend'):
+                logger.warning(
+                    "DEPRECATED: Using `populate_sell_trend()` in the hyperopt file is deprecated. "
+                    "Please move these methods to your strategy."
+                )
+                self.backtesting.strategy.populate_sell_trend = (  # type: ignore
+                    self.custom_hyperopt.populate_sell_trend)  # type: ignore
 
         # Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set
         if self.config.get('use_max_market_positions', True):
@@ -163,13 +180,9 @@ class Hyperopt:
         While not a valid json object - this allows appending easily.
         :param epoch: result dictionary for this epoch.
         """
-        def default_parser(x):
-            if isinstance(x, np.integer):
-                return int(x)
-            return str(x)
-
+        epoch[FTHYPT_FILEVERSION] = 2
         with self.results_file.open('a') as f:
-            rapidjson.dump(epoch, f, default=default_parser,
+            rapidjson.dump(epoch, f, default=hyperopt_serializer,
                            number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN)
             f.write("\n")
 
@@ -191,6 +204,8 @@ class Hyperopt:
             result['buy'] = {p.name: params.get(p.name) for p in self.buy_space}
         if HyperoptTools.has_space(self.config, 'sell'):
             result['sell'] = {p.name: params.get(p.name) for p in self.sell_space}
+        if HyperoptTools.has_space(self.config, 'protection'):
+            result['protection'] = {p.name: params.get(p.name) for p in self.protection_space}
         if HyperoptTools.has_space(self.config, 'roi'):
             result['roi'] = {str(k): v for k, v in
                              self.custom_hyperopt.generate_roi_table(params).items()}
@@ -201,6 +216,25 @@ class Hyperopt:
 
         return result
 
+    def _get_no_optimize_details(self) -> Dict[str, Any]:
+        """
+        Get non-optimized parameters
+        """
+        result: Dict[str, Any] = {}
+        strategy = self.backtesting.strategy
+        if not HyperoptTools.has_space(self.config, 'roi'):
+            result['roi'] = {str(k): v for k, v in strategy.minimal_roi.items()}
+        if not HyperoptTools.has_space(self.config, 'stoploss'):
+            result['stoploss'] = {'stoploss': strategy.stoploss}
+        if not HyperoptTools.has_space(self.config, 'trailing'):
+            result['trailing'] = {
+                'trailing_stop': strategy.trailing_stop,
+                'trailing_stop_positive': strategy.trailing_stop_positive,
+                'trailing_stop_positive_offset': strategy.trailing_stop_positive_offset,
+                'trailing_only_offset_is_reached': strategy.trailing_only_offset_is_reached,
+            }
+        return result
+
     def print_results(self, results) -> None:
         """
         Log results if it is better than any previous evaluation
@@ -222,6 +256,12 @@ class Hyperopt:
         """
         Assign the dimensions in the hyperoptimization space.
         """
+        if self.auto_hyperopt and HyperoptTools.has_space(self.config, 'protection'):
+            # Protections can only be optimized when using the Parameter interface
+            logger.debug("Hyperopt has 'protection' space")
+            # Enable Protections if protection space is selected.
+            self.config['enable_protections'] = True
+            self.protection_space = self.custom_hyperopt.protection_space()
 
         if HyperoptTools.has_space(self.config, 'buy'):
             logger.debug("Hyperopt has 'buy' space")
@@ -242,22 +282,19 @@ class Hyperopt:
         if HyperoptTools.has_space(self.config, 'trailing'):
             logger.debug("Hyperopt has 'trailing' space")
             self.trailing_space = self.custom_hyperopt.trailing_space()
-        self.dimensions = (self.buy_space + self.sell_space + self.roi_space +
-                           self.stoploss_space + self.trailing_space)
+        self.dimensions = (self.buy_space + self.sell_space + self.protection_space
+                           + self.roi_space + self.stoploss_space + self.trailing_space)
 
     def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict:
         """
-        Used Optimize function. Called once per epoch to optimize whatever is configured.
+        Used Optimize function.
+        Called once per epoch to optimize whatever is configured.
         Keep this function as optimized as possible!
         """
         backtest_start_time = datetime.now(timezone.utc)
         params_dict = self._get_params_dict(self.dimensions, raw_params)
 
         # Apply parameters
-        if HyperoptTools.has_space(self.config, 'roi'):
-            self.backtesting.strategy.minimal_roi = (  # type: ignore
-                self.custom_hyperopt.generate_roi_table(params_dict))
-
         if HyperoptTools.has_space(self.config, 'buy'):
             self.backtesting.strategy.advise_buy = (  # type: ignore
                 self.custom_hyperopt.buy_strategy_generator(params_dict))
@@ -266,6 +303,16 @@ class Hyperopt:
             self.backtesting.strategy.advise_sell = (  # type: ignore
                 self.custom_hyperopt.sell_strategy_generator(params_dict))
 
+        if HyperoptTools.has_space(self.config, 'protection'):
+            for attr_name, attr in self.backtesting.strategy.enumerate_parameters('protection'):
+                if attr.optimize:
+                    # noinspection PyProtectedMember
+                    attr.value = params_dict[attr_name]
+
+        if HyperoptTools.has_space(self.config, 'roi'):
+            self.backtesting.strategy.minimal_roi = (  # type: ignore
+                self.custom_hyperopt.generate_roi_table(params_dict))
+
         if HyperoptTools.has_space(self.config, 'stoploss'):
             self.backtesting.strategy.stoploss = params_dict['stoploss']
 
@@ -310,7 +357,8 @@ class Hyperopt:
         results_explanation = HyperoptTools.format_results_explanation_string(
             strat_stats, self.config['stake_currency'])
 
-        not_optimized = self.backtesting.strategy.get_params_dict()
+        not_optimized = self.backtesting.strategy.get_no_optimize_params()
+        not_optimized = deep_merge_dicts(not_optimized, self._get_no_optimize_details())
 
         trade_count = strat_stats['total_trades']
         total_profit = strat_stats['profit_total']
@@ -324,7 +372,8 @@ class Hyperopt:
             loss = self.calculate_loss(results=backtesting_results['results'],
                                        trade_count=trade_count,
                                        min_date=min_date, max_date=max_date,
-                                       config=self.config, processed=processed)
+                                       config=self.config, processed=processed,
+                                       backtest_stats=strat_stats)
         return {
             'loss': loss,
             'params_dict': params_dict,
@@ -357,18 +406,17 @@ class Hyperopt:
         data, timerange = self.backtesting.load_bt_data()
         logger.info("Dataload complete. Calculating indicators")
 
-        preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
+        preprocessed = self.backtesting.strategy.advise_all_indicators(data)
 
-        # Trim startup period from analyzed dataframe
+        # Trim startup period from analyzed dataframe to get correct dates for output.
         processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup)
-
         self.min_date, self.max_date = get_timerange(processed)
 
         logger.info(f'Hyperopting with data from {self.min_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'({(self.max_date - self.min_date).days} days)..')
-
-        dump(processed, self.data_pickle_file)
+        # Store non-trimmed data - will be trimmed after signal generation.
+        dump(preprocessed, self.data_pickle_file)
 
     def start(self) -> None:
         self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
@@ -423,9 +471,9 @@ class Hyperopt:
                         ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
                     ]
                 with progressbar.ProgressBar(
-                         max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
-                         widgets=widgets
-                     ) as pbar:
+                    max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
+                    widgets=widgets
+                ) as pbar:
                     EVALS = ceil(self.total_epochs / jobs)
                     for i in range(EVALS):
                         # Correct the number of epochs to be processed for the last
@@ -469,6 +517,12 @@ class Hyperopt:
                     f"saved to '{self.results_file}'.")
 
         if self.current_best_epoch:
+            if self.auto_hyperopt:
+                HyperoptTools.try_export_params(
+                    self.config,
+                    self.backtesting.strategy.get_strategy_name(),
+                    self.current_best_epoch)
+
             HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs,
                                              self.print_json)
         else:

freqtrade/optimize/hyperopt_auto.py

@@ -73,6 +73,9 @@ class HyperOptAuto(IHyperOpt):
     def sell_indicator_space(self) -> List['Dimension']:
         return self._get_indicator_space('sell', 'sell_indicator_space')
 
+    def protection_space(self) -> List['Dimension']:
+        return self._get_indicator_space('protection', 'protection_space')
+
     def generate_roi_table(self, params: Dict) -> Dict[int, float]:
         return self._get_func('generate_roi_table')(params)
 

freqtrade/optimize/hyperopt_epoch_filters.py

@@ -0,0 +1,128 @@
+import logging
+from typing import List
+
+from freqtrade.exceptions import OperationalException
+
+
+logger = logging.getLogger(__name__)
+
+
+def hyperopt_filter_epochs(epochs: List, filteroptions: dict, log: bool = True) -> List:
+    """
+    Filter our items from the list of hyperopt results
+    """
+    if filteroptions['only_best']:
+        epochs = [x for x in epochs if x['is_best']]
+    if filteroptions['only_profitable']:
+        epochs = [x for x in epochs
+                  if x['results_metrics'].get('profit_total', 0) > 0]
+
+    epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions)
+    if log:
+        logger.info(f"{len(epochs)} " +
+                    ("best " if filteroptions['only_best'] else "") +
+                    ("profitable " if filteroptions['only_profitable'] else "") +
+                    "epochs found.")
+    return epochs
+
+
+def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int):
+    """
+    Filter epochs with trade-counts > trades
+    """
+    return [
+        x for x in epochs if x['results_metrics'].get('total_trades', 0) > trade_count
+    ]
+
+
+def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
+
+    if filteroptions['filter_min_trades'] > 0:
+        epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades'])
+
+    if filteroptions['filter_max_trades'] > 0:
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get('total_trades') < filteroptions['filter_max_trades']
+        ]
+    return epochs
+
+
+def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
+
+    def get_duration_value(x):
+        # Duration in minutes ...
+        if 'holding_avg_s' in x['results_metrics']:
+            avg = x['results_metrics']['holding_avg_s']
+            return avg // 60
+        raise OperationalException(
+            "Holding-average not available. Please omit the filter on average time, "
+            "or rerun hyperopt with this version")
+
+    if filteroptions['filter_min_avg_time'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if get_duration_value(x) > filteroptions['filter_min_avg_time']
+        ]
+    if filteroptions['filter_max_avg_time'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if get_duration_value(x) < filteroptions['filter_max_avg_time']
+        ]
+
+    return epochs
+
+
+def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
+
+    if filteroptions['filter_min_avg_profit'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get('profit_mean', 0) * 100
+            > filteroptions['filter_min_avg_profit']
+        ]
+    if filteroptions['filter_max_avg_profit'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get('profit_mean', 0) * 100
+            < filteroptions['filter_max_avg_profit']
+        ]
+    if filteroptions['filter_min_total_profit'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get('profit_total_abs', 0)
+            > filteroptions['filter_min_total_profit']
+        ]
+    if filteroptions['filter_max_total_profit'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+        epochs = [
+            x for x in epochs
+            if x['results_metrics'].get('profit_total_abs', 0)
+            < filteroptions['filter_max_total_profit']
+        ]
+    return epochs
+
+
+def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
+
+    if filteroptions['filter_min_objective'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+
+        epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
+    if filteroptions['filter_max_objective'] is not None:
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
+
+        epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
+
+    return epochs

freqtrade/optimize/hyperopt_interface.py

@@ -57,6 +57,13 @@ class IHyperOpt(ABC):
         """
         raise OperationalException(_format_exception_message('sell_strategy_generator', 'sell'))
 
+    def protection_space(self) -> List[Dimension]:
+        """
+        Create a protection space.
+        Only supported by the Parameter interface.
+        """
+        raise OperationalException(_format_exception_message('indicator_space', 'protection'))
+
     def indicator_space(self) -> List[Dimension]:
         """
         Create an indicator space.

freqtrade/optimize/hyperopt_loss_interface.py

@@ -5,7 +5,7 @@ This module defines the interface for the loss-function for hyperopt
 
 from abc import ABC, abstractmethod
 from datetime import datetime
-from typing import Dict
+from typing import Any, Dict
 
 from pandas import DataFrame
 
@@ -22,6 +22,7 @@ class IHyperOptLoss(ABC):
     def hyperopt_loss_function(results: DataFrame, trade_count: int,
                                min_date: datetime, max_date: datetime,
                                config: Dict, processed: Dict[str, DataFrame],
+                               backtest_stats: Dict[str, Any],
                                *args, **kwargs) -> float:
         """
         Objective function, returns smaller number for better results

freqtrade/optimize/hyperopt_tools.py

@@ -1,75 +1,159 @@
 
 import io
 import logging
+from copy import deepcopy
+from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any, Dict, Iterator, List, Optional, Tuple
 
+import numpy as np
 import rapidjson
 import tabulate
 from colorama import Fore, Style
 from pandas import isna, json_normalize
 
+from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES
 from freqtrade.exceptions import OperationalException
-from freqtrade.misc import round_coin_value, round_dict
+from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2
+from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs
 
 
 logger = logging.getLogger(__name__)
 
+NON_OPT_PARAM_APPENDIX = "  # value loaded from strategy"
+
+
+def hyperopt_serializer(x):
+    if isinstance(x, np.integer):
+        return int(x)
+    if isinstance(x, np.bool_):
+        return bool(x)
+
+    return str(x)
+
 
 class HyperoptTools():
 
+    @staticmethod
+    def get_strategy_filename(config: Dict, strategy_name: str) -> Optional[Path]:
+        """
+        Get Strategy-location (filename) from strategy_name
+        """
+        from freqtrade.resolvers.strategy_resolver import StrategyResolver
+        directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES))
+        strategy_objs = StrategyResolver.search_all_objects(directory, False)
+        strategies = [s for s in strategy_objs if s['name'] == strategy_name]
+        if strategies:
+            strategy = strategies[0]
+
+            return Path(strategy['location'])
+        return None
+
+    @staticmethod
+    def export_params(params, strategy_name: str, filename: Path):
+        """
+        Generate files
+        """
+        final_params = deepcopy(params['params_not_optimized'])
+        final_params = deep_merge_dicts(params['params_details'], final_params)
+        final_params = {
+            'strategy_name': strategy_name,
+            'params': final_params,
+            'ft_stratparam_v': 1,
+            'export_time': datetime.now(timezone.utc),
+        }
+        logger.info(f"Dumping parameters to {filename}")
+        rapidjson.dump(final_params, filename.open('w'), indent=2,
+                       default=hyperopt_serializer,
+                       number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN
+                       )
+
+    @staticmethod
+    def try_export_params(config: Dict[str, Any], strategy_name: str, params: Dict):
+        if params.get(FTHYPT_FILEVERSION, 1) >= 2 and not config.get('disableparamexport', False):
+            # Export parameters ...
+            fn = HyperoptTools.get_strategy_filename(config, strategy_name)
+            if fn:
+                HyperoptTools.export_params(params, strategy_name, fn.with_suffix('.json'))
+            else:
+                logger.warning("Strategy not found, not exporting parameter file.")
+
     @staticmethod
     def has_space(config: Dict[str, Any], space: str) -> bool:
         """
         Tell if the space value is contained in the configuration
         """
-        # The 'trailing' space is not included in the 'default' set of spaces
-        if space == 'trailing':
+        # 'trailing' and 'protection spaces are not included in the 'default' set of spaces
+        if space in ('trailing', 'protection'):
             return any(s in config['spaces'] for s in [space, 'all'])
         else:
             return any(s in config['spaces'] for s in [space, 'all', 'default'])
 
     @staticmethod
-    def _read_results_pickle(results_file: Path) -> List:
+    def _read_results(results_file: Path, batch_size: int = 10) -> Iterator[List[Any]]:
         """
-        Read hyperopt results from pickle file
-        LEGACY method - new files are written as json and cannot be read with this method.
-        """
-        from joblib import load
-
-        logger.info(f"Reading pickled epochs from '{results_file}'")
-        data = load(results_file)
-        return data
-
-    @staticmethod
-    def _read_results(results_file: Path) -> List:
-        """
-        Read hyperopt results from file
+        Stream hyperopt results from file
         """
         import rapidjson
         logger.info(f"Reading epochs from '{results_file}'")
         with results_file.open('r') as f:
-            data = [rapidjson.loads(line) for line in f]
-        return data
+            data = []
+            for line in f:
+                data += [rapidjson.loads(line)]
+                if len(data) >= batch_size:
+                    yield data
+                    data = []
+        yield data
 
     @staticmethod
-    def load_previous_results(results_file: Path) -> List:
-        """
-        Load data for epochs from the file if we have one
-        """
-        epochs: List = []
+    def _test_hyperopt_results_exist(results_file) -> bool:
         if results_file.is_file() and results_file.stat().st_size > 0:
             if results_file.suffix == '.pickle':
-                epochs = HyperoptTools._read_results_pickle(results_file)
-            else:
-                epochs = HyperoptTools._read_results(results_file)
-            # Detection of some old format, without 'is_best' field saved
-            if epochs[0].get('is_best') is None:
+                raise OperationalException(
+                    "Legacy hyperopt results are no longer supported."
+                    "Please rerun hyperopt or use an older version to load this file."
+                )
+            return True
+        else:
+            # No file found.
+            return False
+
+    @staticmethod
+    def load_filtered_results(results_file: Path, config: Dict[str, Any]) -> Tuple[List, int]:
+        filteroptions = {
+            'only_best': config.get('hyperopt_list_best', False),
+            'only_profitable': config.get('hyperopt_list_profitable', False),
+            'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
+            'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
+            'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
+            'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
+            'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
+            'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
+            'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
+            'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
+            'filter_min_objective': config.get('hyperopt_list_min_objective', None),
+            'filter_max_objective': config.get('hyperopt_list_max_objective', None),
+        }
+        if not HyperoptTools._test_hyperopt_results_exist(results_file):
+            # No file found.
+            return [], 0
+
+        epochs = []
+        total_epochs = 0
+        for epochs_tmp in HyperoptTools._read_results(results_file):
+            if total_epochs == 0 and epochs_tmp[0].get('is_best') is None:
                 raise OperationalException(
                     "The file with HyperoptTools results is incompatible with this version "
                     "of Freqtrade and cannot be loaded.")
-            logger.info(f"Loaded {len(epochs)} previous evaluations from disk.")
-        return epochs
+            total_epochs += len(epochs_tmp)
+            epochs += hyperopt_filter_epochs(epochs_tmp, filteroptions, log=False)
+
+        logger.info(f"Loaded {total_epochs} previous evaluations from disk.")
+
+        # Final filter run ...
+        epochs = hyperopt_filter_epochs(epochs, filteroptions, log=True)
+
+        return epochs, total_epochs
 
     @staticmethod
     def show_epoch_details(results, total_epochs: int, print_json: bool,
@@ -90,7 +174,7 @@ class HyperoptTools():
 
         if print_json:
             result_dict: Dict = {}
-            for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']:
+            for s in ['buy', 'sell', 'protection', 'roi', 'stoploss', 'trailing']:
                 HyperoptTools._params_update_for_json(result_dict, params, non_optimized, s)
             print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE))
 
@@ -99,9 +183,11 @@ class HyperoptTools():
                                                non_optimized)
             HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:",
                                                non_optimized)
-            HyperoptTools._params_pretty_print(params, 'roi', "ROI table:")
-            HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:")
-            HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:")
+            HyperoptTools._params_pretty_print(params, 'protection',
+                                               "Protection hyperspace params:", non_optimized)
+            HyperoptTools._params_pretty_print(params, 'roi', "ROI table:", non_optimized)
+            HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:", non_optimized)
+            HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:", non_optimized)
 
     @staticmethod
     def _params_update_for_json(result_dict, params, non_optimized, space: str) -> None:
@@ -127,23 +213,34 @@ class HyperoptTools():
     def _params_pretty_print(params, space: str, header: str, non_optimized={}) -> None:
         if space in params or space in non_optimized:
             space_params = HyperoptTools._space_params(params, space, 5)
+            no_params = HyperoptTools._space_params(non_optimized, space, 5)
+            appendix = ''
+            if not space_params and not no_params:
+                # No parameters - don't print
+                return
+            if not space_params:
+                # Not optimized parameters - append string
+                appendix = NON_OPT_PARAM_APPENDIX
+
             result = f"\n# {header}\n"
-            if space == 'stoploss':
-                result += f"stoploss = {space_params.get('stoploss')}"
-            elif space == 'roi':
+            if space == "stoploss":
+                stoploss = safe_value_fallback2(space_params, no_params, space, space)
+                result += (f"stoploss = {stoploss}{appendix}")
+
+            elif space == "roi":
+                result = result[:-1] + f'{appendix}\n'
                 minimal_roi_result = rapidjson.dumps({
-                        str(k): v for k, v in space_params.items()
+                    str(k): v for k, v in (space_params or no_params).items()
                 }, default=str, indent=4, number_mode=rapidjson.NM_NATIVE)
                 result += f"minimal_roi = {minimal_roi_result}"
-            elif space == 'trailing':
-
-                for k, v in space_params.items():
-                    result += f'{k} = {v}\n'
+            elif space == "trailing":
+                for k, v in (space_params or no_params).items():
+                    result += f"{k} = {v}{appendix}\n"
 
             else:
-                no_params = HyperoptTools._space_params(non_optimized, space, 5)
+                # Buy / sell parameters
 
-                result += f"{space}_params = {HyperoptTools._pprint(space_params, no_params)}"
+                result += f"{space}_params = {HyperoptTools._pprint_dict(space_params, no_params)}"
 
             result = result.replace("\n", "\n    ")
             print(result)
@@ -157,7 +254,7 @@ class HyperoptTools():
         return {}
 
     @staticmethod
-    def _pprint(params, non_optimized, indent: int = 4):
+    def _pprint_dict(params, non_optimized, indent: int = 4):
         """
         Pretty-print hyperopt results (based on 2 dicts - with add. comment)
         """
@@ -169,7 +266,7 @@ class HyperoptTools():
             result += " " * indent + f'"{k}": '
             result += f'"{param}",' if isinstance(param, str) else f'{param},'
             if k in non_optimized:
-                result += "  # value loaded from strategy"
+                result += NON_OPT_PARAM_APPENDIX
             result += "\n"
         result += '}'
         return result
@@ -361,21 +458,14 @@ class HyperoptTools():
         trials['Best'] = ''
         trials['Stake currency'] = config['stake_currency']
 
-        if 'results_metrics.total_trades' in trials:
-            base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades',
-                            'results_metrics.profit_mean', 'results_metrics.profit_median',
-                            'results_metrics.profit_total',
-                            'Stake currency',
-                            'results_metrics.profit_total_abs', 'results_metrics.holding_avg',
-                            'loss', 'is_initial_point', 'is_best']
-            perc_multi = 100
-        else:
-            perc_multi = 1
-            base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count',
-                            'results_metrics.avg_profit', 'results_metrics.median_profit',
-                            'results_metrics.total_profit',
-                            'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
-                            'loss', 'is_initial_point', 'is_best']
+        base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades',
+                        'results_metrics.profit_mean', 'results_metrics.profit_median',
+                        'results_metrics.profit_total',
+                        'Stake currency',
+                        'results_metrics.profit_total_abs', 'results_metrics.holding_avg',
+                        'loss', 'is_initial_point', 'is_best']
+        perc_multi = 100
+
         param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()]
         trials = trials[base_metrics + param_metrics]
 
@@ -403,11 +493,6 @@ class HyperoptTools():
         trials['Avg profit'] = trials['Avg profit'].apply(
             lambda x: f'{x * perc_multi:,.2f}%' if not isna(x) else ""
         )
-        if perc_multi == 1:
-            trials['Avg duration'] = trials['Avg duration'].apply(
-                lambda x: f'{x:,.1f} m' if isinstance(
-                    x, float) else f"{x.total_seconds() // 60:,.1f} m" if not isna(x) else ""
-            )
         trials['Objective'] = trials['Objective'].apply(
             lambda x: f'{x:,.5f}' if x != 100000 else ""
         )

freqtrade/optimize/optimize_reports.py

@@ -31,7 +31,7 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N
         filename = Path.joinpath(
             recordfilename.parent,
             f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}'
-            ).with_suffix(recordfilename.suffix)
+        ).with_suffix(recordfilename.suffix)
     file_dump_json(filename, stats)
 
     latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN)
@@ -173,7 +173,7 @@ def generate_strategy_comparison(all_results: Dict) -> List[Dict]:
     for strategy, results in all_results.items():
         tabular_data.append(_generate_result_line(
             results['results'], results['config']['dry_run_wallet'], strategy)
-            )
+        )
         try:
             max_drawdown_per, _, _, _, _ = calculate_max_drawdown(results['results'],
                                                                   value_col='profit_ratio')
@@ -229,8 +229,6 @@ def generate_trading_stats(results: DataFrame) -> Dict[str, Any]:
     winning_trades = results.loc[results['profit_ratio'] > 0]
     draw_trades = results.loc[results['profit_ratio'] == 0]
     losing_trades = results.loc[results['profit_ratio'] < 0]
-    zero_duration_trades = len(results.loc[(results['trade_duration'] == 0) &
-                                           (results['sell_reason'] == 'trailing_stop_loss')])
 
     holding_avg = (timedelta(minutes=round(results['trade_duration'].mean()))
                    if not results.empty else timedelta())
@@ -249,7 +247,6 @@ def generate_trading_stats(results: DataFrame) -> Dict[str, Any]:
         'winner_holding_avg_s': winner_holding_avg.total_seconds(),
         'loser_holding_avg': loser_holding_avg,
         'loser_holding_avg_s': loser_holding_avg.total_seconds(),
-        'zero_duration_trades': zero_duration_trades,
     }
 
 
@@ -264,6 +261,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
             'winning_days': 0,
             'draw_days': 0,
             'losing_days': 0,
+            'daily_profit_list': [],
         }
     daily_profit_rel = results.resample('1d', on='close_date')['profit_ratio'].sum()
     daily_profit = results.resample('1d', on='close_date')['profit_abs'].sum().round(10)
@@ -274,6 +272,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
     winning_days = sum(daily_profit > 0)
     draw_days = sum(daily_profit == 0)
     losing_days = sum(daily_profit < 0)
+    daily_profit_list = [(str(idx.date()), val) for idx, val in daily_profit.iteritems()]
 
     return {
         'backtest_best_day': best_rel,
@@ -283,6 +282,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
         'winning_days': winning_days,
         'draw_days': draw_days,
         'losing_days': losing_days,
+        'daily_profit': daily_profit_list,
     }
 
 
@@ -325,8 +325,9 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
                     key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
     worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'],
                      key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
-    results['open_timestamp'] = results['open_date'].astype(int64) // 1e6
-    results['close_timestamp'] = results['close_date'].astype(int64) // 1e6
+    if not results.empty:
+        results['open_timestamp'] = results['open_date'].view(int64) // 1e6
+        results['close_timestamp'] = results['close_date'].view(int64) // 1e6
 
     backtest_days = (max_date - min_date).days
     strat_stats = {
@@ -378,10 +379,10 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
         'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False),
         'use_custom_stoploss': config.get('use_custom_stoploss', False),
         'minimal_roi': config['minimal_roi'],
-        'use_sell_signal': config['ask_strategy']['use_sell_signal'],
-        'sell_profit_only': config['ask_strategy']['sell_profit_only'],
-        'sell_profit_offset': config['ask_strategy']['sell_profit_offset'],
-        'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'],
+        'use_sell_signal': config['use_sell_signal'],
+        'sell_profit_only': config['sell_profit_only'],
+        'sell_profit_offset': config['sell_profit_offset'],
+        'ignore_roi_if_buy_signal': config['ignore_roi_if_buy_signal'],
         **daily_stats,
         **trade_stats
     }
@@ -542,14 +543,6 @@ def text_table_add_metrics(strat_results: Dict) -> str:
         # Newly added fields should be ignored if they are missing in strat_results. hyperopt-show
         # command stores these results and newer version of freqtrade must be able to handle old
         # results with missing new fields.
-        zero_duration_trades = '--'
-
-        if 'zero_duration_trades' in strat_results:
-            zero_duration_trades_per = \
-                100.0 / strat_results['total_trades'] * strat_results['zero_duration_trades']
-            zero_duration_trades = f'{zero_duration_trades_per:.2f}% ' \
-                                   f'({strat_results["zero_duration_trades"]})'
-
         metrics = [
             ('Backtesting from', strat_results['backtest_start']),
             ('Backtesting to', strat_results['backtest_end']),
@@ -585,7 +578,6 @@ def text_table_add_metrics(strat_results: Dict) -> str:
                 f"{strat_results['draw_days']} / {strat_results['losing_days']}"),
             ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"),
             ('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"),
-            ('Zero Duration Trades', zero_duration_trades),
             ('Rejected Buy signals', strat_results.get('rejected_signals', 'N/A')),
             ('', ''),  # Empty line to improve readability
 
@@ -612,7 +604,7 @@ def text_table_add_metrics(strat_results: Dict) -> str:
                                          strat_results['stake_currency'])
         stake_amount = round_coin_value(
             strat_results['stake_amount'], strat_results['stake_currency']
-            ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited'
+        ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited'
 
         message = ("No trades made. "
                    f"Your starting balance was {start_balance}, "
@@ -663,6 +655,8 @@ def show_backtest_results(config: Dict, backtest_stats: Dict):
         # Print Strategy summary table
 
         table = text_table_strategy(backtest_stats['strategy_comparison'], stake_currency)
+        print(f"{results['backtest_start']} -> {results['backtest_end']} |"
+              f" Max open trades : {results['max_open_trades']}")
         print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '='))
         print(table)
         print('=' * len(table.splitlines()[0]))

freqtrade/persistence/migrations.py

@@ -47,6 +47,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
     min_rate = get_column_def(cols, 'min_rate', 'null')
     sell_reason = get_column_def(cols, 'sell_reason', 'null')
     strategy = get_column_def(cols, 'strategy', 'null')
+    buy_tag = get_column_def(cols, 'buy_tag', 'null')
     # If ticker-interval existed use that, else null.
     if has_column(cols, 'ticker_interval'):
         timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
@@ -64,7 +65,8 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
     # Schema migration necessary
     with engine.begin() as connection:
         connection.execute(text(f"alter table trades rename to {table_back_name}"))
-        # drop indexes on backup table
+    with engine.begin() as connection:
+        # drop indexes on backup table in new session
         for index in inspector.get_indexes(table_back_name):
             connection.execute(text(f"drop index {index['name']}"))
     # let SQLAlchemy create the schema as required
@@ -75,22 +77,15 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
         connection.execute(text(f"""insert into trades
             (id, exchange, pair, is_open,
             fee_open, fee_open_cost, fee_open_currency,
-            fee_close, fee_close_cost, fee_open_currency, open_rate,
+            fee_close, fee_close_cost, fee_close_currency, open_rate,
             open_rate_requested, close_rate, close_rate_requested, close_profit,
             stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
             stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
             stoploss_order_id, stoploss_last_update,
-            max_rate, min_rate, sell_reason, sell_order_status, strategy,
+            max_rate, min_rate, sell_reason, sell_order_status, strategy, buy_tag,
             timeframe, open_trade_value, close_profit_abs
             )
-        select id, lower(exchange),
-            case
-                when instr(pair, '_') != 0 then
-                substr(pair,    instr(pair, '_') + 1) || '/' ||
-                substr(pair, 1, instr(pair, '_') - 1)
-                else pair
-                end
-            pair,
+        select id, lower(exchange), pair,
             is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
             {fee_open_currency} fee_open_currency, {fee_close} fee_close,
             {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
@@ -103,7 +98,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
             {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
             {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
             {sell_order_status} sell_order_status,
-            {strategy} strategy, {timeframe} timeframe,
+            {strategy} strategy, {buy_tag} buy_tag, {timeframe} timeframe,
             {open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs
             from {table_back_name}
             """))
@@ -131,7 +126,9 @@ def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, col
 
     with engine.begin() as connection:
         connection.execute(text(f"alter table orders rename to {table_back_name}"))
-        # drop indexes on backup table
+
+    with engine.begin() as connection:
+        # drop indexes on backup table in new session
         for index in inspector.get_indexes(table_back_name):
             connection.execute(text(f"drop index {index['name']}"))
 
@@ -160,7 +157,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
     table_back_name = get_backup_name(tabs, 'trades_bak')
 
     # Check for latest column
-    if not has_column(cols, 'open_trade_value'):
+    if not has_column(cols, 'buy_tag'):
         logger.info(f'Running database migration for trades - backup: {table_back_name}')
         migrate_trades_table(decl_base, inspector, engine, table_back_name, cols)
         # Reread columns - the above recreated the table!

freqtrade/persistence/models.py

@@ -13,7 +13,7 @@ from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session
 from sqlalchemy.pool import StaticPool
 from sqlalchemy.sql.schema import UniqueConstraint
 
-from freqtrade.constants import DATETIME_PRINT_FORMAT
+from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES
 from freqtrade.enums import SellType
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.misc import safe_value_fallback
@@ -159,9 +159,9 @@ class Order(_DECL_BASE):
             self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc)
 
         self.ft_is_open = True
-        if self.status in ('closed', 'canceled', 'cancelled'):
+        if self.status in NON_OPEN_EXCHANGE_STATES:
             self.ft_is_open = False
-            if order.get('filled', 0) > 0:
+            if (order.get('filled', 0.0) or 0.0) > 0:
                 self.order_filled_date = datetime.now(timezone.utc)
         self.order_update_date = datetime.now(timezone.utc)
 
@@ -257,6 +257,7 @@ class LocalTrade():
     sell_reason: str = ''
     sell_order_status: str = ''
     strategy: str = ''
+    buy_tag: Optional[str] = None
     timeframe: Optional[int] = None
 
     def __init__(self, **kwargs):
@@ -288,6 +289,7 @@ class LocalTrade():
             'amount_requested': round(self.amount_requested, 8) if self.amount_requested else None,
             'stake_amount': round(self.stake_amount, 8),
             'strategy': self.strategy,
+            'buy_tag': self.buy_tag,
             'timeframe': self.timeframe,
 
             'fee_open': self.fee_open,
@@ -352,12 +354,12 @@ class LocalTrade():
         LocalTrade.trades_open = []
         LocalTrade.total_profit = 0
 
-    def adjust_min_max_rates(self, current_price: float) -> None:
+    def adjust_min_max_rates(self, current_price: float, current_price_low: float) -> None:
         """
         Adjust the max_rate and min_rate.
         """
         self.max_rate = max(current_price, self.max_rate or self.open_rate)
-        self.min_rate = min(current_price, self.min_rate or self.open_rate)
+        self.min_rate = min(current_price_low, self.min_rate or self.open_rate)
 
     def _set_new_stoploss(self, new_loss: float, stoploss: float):
         """Assign new stop value"""
@@ -636,7 +638,7 @@ class LocalTrade():
 
             # skip case if trailing-stop changed the stoploss already.
             if (trade.stop_loss == trade.initial_stop_loss
-               and trade.initial_stop_loss_pct != desired_stoploss):
+                    and trade.initial_stop_loss_pct != desired_stoploss):
                 # Stoploss value got changed
 
                 logger.info(f"Stoploss for {trade} needs adjustment...")
@@ -703,6 +705,7 @@ class Trade(_DECL_BASE, LocalTrade):
     sell_reason = Column(String(100), nullable=True)
     sell_order_status = Column(String(100), nullable=True)
     strategy = Column(String(100), nullable=True)
+    buy_tag = Column(String(100), nullable=True)
     timeframe = Column(Integer, nullable=True)
 
     def __init__(self, **kwargs):
@@ -801,6 +804,19 @@ class Trade(_DECL_BASE, LocalTrade):
                                  Trade.is_open.is_(False),
                                  ]).all()
 
+    @staticmethod
+    def get_total_closed_profit() -> float:
+        """
+        Retrieves total realized profit
+        """
+        if Trade.use_db:
+            total_profit = Trade.query.with_entities(
+                func.sum(Trade.close_profit_abs)).filter(Trade.is_open.is_(False)).scalar()
+        else:
+            total_profit = sum(
+                t.close_profit_abs for t in LocalTrade.get_trades_proxy(is_open=False))
+        return total_profit or 0
+
     @staticmethod
     def total_open_trades_stakes() -> float:
         """
@@ -841,7 +857,7 @@ class Trade(_DECL_BASE, LocalTrade):
         ]
 
     @staticmethod
-    def get_best_pair():
+    def get_best_pair(start_date: datetime = datetime.fromtimestamp(0)):
         """
         Get best pair with closed trade.
         NOTE: Not supported in Backtesting.
@@ -849,7 +865,7 @@ class Trade(_DECL_BASE, LocalTrade):
         """
         best_pair = Trade.query.with_entities(
             Trade.pair, func.sum(Trade.close_profit).label('profit_sum')
-        ).filter(Trade.is_open.is_(False)) \
+        ).filter(Trade.is_open.is_(False) & (Trade.close_date >= start_date)) \
             .group_by(Trade.pair) \
             .order_by(desc('profit_sum')).first()
         return best_pair

freqtrade/plot/plotting.py

@@ -334,8 +334,8 @@ def add_areas(fig, row: int, data: pd.DataFrame, indicators) -> make_subplots:
                 )
             elif indicator_b not in data:
                 logger.info(
-                        'fill_to: "%s" ignored. Reason: This indicator is not '
-                        'in your strategy.', indicator_b
+                    'fill_to: "%s" ignored. Reason: This indicator is not '
+                    'in your strategy.', indicator_b
                 )
     return fig
 
@@ -373,6 +373,7 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra
     for i, name in enumerate(plot_config['subplots']):
         fig['layout'][f'yaxis{3 + i}'].update(title=name)
     fig['layout']['xaxis']['rangeslider'].update(visible=False)
+    fig.update_layout(modebar_add=["v1hovermode", "toggleSpikeLines"])
 
     # Common information
     candles = go.Candlestick(
@@ -452,6 +453,7 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra
                              data=data)
         # fill area between indicators ( 'fill_to': 'other_indicator')
         fig = add_areas(fig, row, data, sub_config)
+
     return fig
 
 
@@ -484,6 +486,7 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
     fig['layout']['yaxis2'].update(title=f'Profit {stake_currency}')
     fig['layout']['yaxis3'].update(title=f'Profit {stake_currency}')
     fig['layout']['xaxis']['rangeslider'].update(visible=False)
+    fig.update_layout(modebar_add=["v1hovermode", "toggleSpikeLines"])
 
     fig.add_trace(avgclose, 1, 1)
     fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit')
@@ -497,7 +500,6 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
             fig = add_profit(fig, 3, df_comb, profit_col, f"Profit {pair}")
         except ValueError:
             pass
-
     return fig
 
 
@@ -536,7 +538,7 @@ def load_and_plot_trades(config: Dict[str, Any]):
     - Initializes plot-script
     - Get candle (OHLCV) data
     - Generate Dafaframes populated with indicators and signals based on configured strategy
-    - Load trades excecuted during the selected period
+    - Load trades executed during the selected period
     - Generate Plotly plot objects
     - Generate plot files
     :return: None

freqtrade/plugins/pairlist/AgeFilter.py

@@ -27,6 +27,7 @@ class AgeFilter(IPairList):
         super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
 
         self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
+        self._max_days_listed = pairlistconfig.get('max_days_listed', None)
 
         if self._min_days_listed < 1:
             raise OperationalException("AgeFilter requires min_days_listed to be >= 1")
@@ -34,6 +35,12 @@ class AgeFilter(IPairList):
             raise OperationalException("AgeFilter requires min_days_listed to not exceed "
                                        "exchange max request size "
                                        f"({exchange.ohlcv_candle_limit('1d')})")
+        if self._max_days_listed and self._max_days_listed <= self._min_days_listed:
+            raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted")
+        if self._max_days_listed and self._max_days_listed > exchange.ohlcv_candle_limit('1d'):
+            raise OperationalException("AgeFilter requires max_days_listed to not exceed "
+                                       "exchange max request size "
+                                       f"({exchange.ohlcv_candle_limit('1d')})")
 
     @property
     def needstickers(self) -> bool:
@@ -48,8 +55,13 @@ class AgeFilter(IPairList):
         """
         Short whitelist method description - used for startup-messages
         """
-        return (f"{self.name} - Filtering pairs with age less than "
-                f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.")
+        return (
+            f"{self.name} - Filtering pairs with age less than "
+            f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}"
+        ) + ((
+            " or more than "
+            f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
+        ) if self._max_days_listed else '')
 
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
@@ -61,9 +73,12 @@ class AgeFilter(IPairList):
         if not needed_pairs:
             return pairlist
 
+        since_days = -(
+            self._max_days_listed if self._max_days_listed else self._min_days_listed
+        ) - 1
         since_ms = int(arrow.utcnow()
                        .floor('day')
-                       .shift(days=-self._min_days_listed - 1)
+                       .shift(days=since_days)
                        .float_timestamp) * 1000
         candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
         if self._enabled:
@@ -86,14 +101,22 @@ class AgeFilter(IPairList):
             return True
 
         if daily_candles is not None:
-            if len(daily_candles) >= self._min_days_listed:
+            if (
+                len(daily_candles) >= self._min_days_listed
+                and (not self._max_days_listed or len(daily_candles) <= self._max_days_listed)
+            ):
                 # We have fetched at least the minimum required number of daily candles
                 # Add to cache, store the time we last checked this symbol
-                self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000
+                self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000
                 return True
             else:
-                self.log_once(f"Removed {pair} from whitelist, because age "
-                              f"{len(daily_candles)} is less than {self._min_days_listed} "
-                              f"{plural(self._min_days_listed, 'day')}", logger.info)
+                self.log_once((
+                    f"Removed {pair} from whitelist, because age "
+                    f"{len(daily_candles)} is less than {self._min_days_listed} "
+                    f"{plural(self._min_days_listed, 'day')}"
+                ) + ((
+                    " or more than "
+                    f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
+                ) if self._max_days_listed else ''), logger.info)
                 return False
         return False

freqtrade/plugins/pairlist/IPairList.py

@@ -144,24 +144,26 @@ class IPairList(LoggingMixin, ABC):
         markets = self._exchange.markets
         if not markets:
             raise OperationalException(
-                    'Markets not loaded. Make sure that exchange is initialized correctly.')
+                'Markets not loaded. Make sure that exchange is initialized correctly.')
 
         sanitized_whitelist: List[str] = []
         for pair in pairlist:
             # pair is not in the generated dynamic market or has the wrong stake currency
             if pair not in markets:
-                logger.warning(f"Pair {pair} is not compatible with exchange "
-                               f"{self._exchange.name}. Removing it from whitelist..")
+                self.log_once(f"Pair {pair} is not compatible with exchange "
+                              f"{self._exchange.name}. Removing it from whitelist..",
+                              logger.warning)
                 continue
 
             if not self._exchange.market_is_tradable(markets[pair]):
-                logger.warning(f"Pair {pair} is not tradable with Freqtrade."
-                               "Removing it from whitelist..")
+                self.log_once(f"Pair {pair} is not tradable with Freqtrade."
+                              "Removing it from whitelist..", logger.warning)
                 continue
 
             if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']:
-                logger.warning(f"Pair {pair} is not compatible with your stake currency "
-                               f"{self._config['stake_currency']}. Removing it from whitelist..")
+                self.log_once(f"Pair {pair} is not compatible with your stake currency "
+                              f"{self._config['stake_currency']}. Removing it from whitelist..",
+                              logger.warning)
                 continue
 
             # Check if market is active

freqtrade/plugins/pairlist/OffsetFilter.py

@@ -0,0 +1,54 @@
+"""
+Offset pair list filter
+"""
+import logging
+from typing import Any, Dict, List
+
+from freqtrade.exceptions import OperationalException
+from freqtrade.plugins.pairlist.IPairList import IPairList
+
+
+logger = logging.getLogger(__name__)
+
+
+class OffsetFilter(IPairList):
+
+    def __init__(self, exchange, pairlistmanager,
+                 config: Dict[str, Any], pairlistconfig: Dict[str, Any],
+                 pairlist_pos: int) -> None:
+        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+
+        self._offset = pairlistconfig.get('offset', 0)
+
+        if self._offset < 0:
+            raise OperationalException("OffsetFilter requires offset to be >= 0")
+
+    @property
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requires tickers, an empty Dict is passed
+        as tickers argument to filter_pairlist
+        """
+        return False
+
+    def short_desc(self) -> str:
+        """
+        Short whitelist method description - used for startup-messages
+        """
+        return f"{self.name} - Offseting pairs by {self._offset}."
+
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Filters and sorts pairlist and returns the whitelist again.
+        Called on each bot iteration - please use internal caching if necessary
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new whitelist
+        """
+        if self._offset > len(pairlist):
+            self.log_once(f"Offset of {self._offset} is larger than " +
+                          f"pair count of {len(pairlist)}", logger.warning)
+        pairs = pairlist[self._offset:]
+        self.log_once(f"Searching {len(pairs)} pairs: {pairs}", logger.info)
+        return pairs

freqtrade/plugins/pairlist/VolatilityFilter.py

@@ -69,10 +69,10 @@ class VolatilityFilter(IPairList):
         """
         needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
 
-        since_ms = int(arrow.utcnow()
-                       .floor('day')
-                       .shift(days=-self._days - 1)
-                       .float_timestamp) * 1000
+        since_ms = (arrow.utcnow()
+                         .floor('day')
+                         .shift(days=-self._days - 1)
+                         .int_timestamp) * 1000
         # Get all candles
         candles = {}
         if needed_pairs:

freqtrade/plugins/pairlist/VolumePairList.py

@@ -4,11 +4,15 @@ Volume PairList provider
 Provides dynamic pair list based on trade volumes
 """
 import logging
+from functools import partial
 from typing import Any, Dict, List
 
+import arrow
 from cachetools.ttl import TTLCache
 
 from freqtrade.exceptions import OperationalException
+from freqtrade.exchange import timeframe_to_minutes
+from freqtrade.misc import format_ms_time
 from freqtrade.plugins.pairlist.IPairList import IPairList
 
 
@@ -36,6 +40,35 @@ class VolumePairList(IPairList):
         self._min_value = self._pairlistconfig.get('min_value', 0)
         self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
         self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
+        self._lookback_days = self._pairlistconfig.get('lookback_days', 0)
+        self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d')
+        self._lookback_period = self._pairlistconfig.get('lookback_period', 0)
+
+        if (self._lookback_days > 0) & (self._lookback_period > 0):
+            raise OperationalException(
+                'Ambigous configuration: lookback_days and lookback_period both set in pairlist '
+                'config. Please set lookback_days only or lookback_period and lookback_timeframe '
+                'and restart the bot.'
+            )
+
+        # overwrite lookback timeframe and days when lookback_days is set
+        if self._lookback_days > 0:
+            self._lookback_timeframe = '1d'
+            self._lookback_period = self._lookback_days
+
+        # get timeframe in minutes and seconds
+        self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
+        self._tf_in_sec = self._tf_in_min * 60
+
+        # wether to use range lookback or not
+        self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
+
+        if self._use_range & (self._refresh_period < self._tf_in_sec):
+            raise OperationalException(
+                f'Refresh period of {self._refresh_period} seconds is smaller than one '
+                f'timeframe of {self._lookback_timeframe}. Please adjust refresh_period '
+                f'to at least {self._tf_in_sec} and restart the bot.'
+            )
 
         if not self._exchange.exchange_has('fetchTickers'):
             raise OperationalException(
@@ -47,6 +80,13 @@ class VolumePairList(IPairList):
             raise OperationalException(
                 f'key {self._sort_key} not in {SORT_VALUES}')
 
+        if self._lookback_period < 0:
+            raise OperationalException("VolumeFilter requires lookback_period to be >= 0")
+        if self._lookback_period > exchange.ohlcv_candle_limit(self._lookback_timeframe):
+            raise OperationalException("VolumeFilter requires lookback_period to not "
+                                       "exceed exchange max request size "
+                                       f"({exchange.ohlcv_candle_limit(self._lookback_timeframe)})")
+
     @property
     def needstickers(self) -> bool:
         """
@@ -76,19 +116,18 @@ class VolumePairList(IPairList):
         pairlist = self._pair_cache.get('pairlist')
         if pairlist:
             # Item found - no refresh necessary
-            return pairlist
+            return pairlist.copy()
         else:
-
             # Use fresh pairlist
             # Check if pair quote currency equals to the stake currency.
             filtered_tickers = [
-                    v for k, v in tickers.items()
-                    if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
-                        and v[self._sort_key] is not None)]
+                v for k, v in tickers.items()
+                if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
+                    and v[self._sort_key] is not None)]
             pairlist = [s['symbol'] for s in filtered_tickers]
 
             pairlist = self.filter_pairlist(pairlist, tickers)
-            self._pair_cache['pairlist'] = pairlist
+            self._pair_cache['pairlist'] = pairlist.copy()
 
         return pairlist
 
@@ -103,15 +142,69 @@ class VolumePairList(IPairList):
         # Use the incoming pairlist.
         filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
 
+        # get lookback period in ms, for exchange ohlcv fetch
+        if self._use_range:
+            since_ms = int(arrow.utcnow()
+                           .floor('minute')
+                           .shift(minutes=-(self._lookback_period * self._tf_in_min)
+                                  - self._tf_in_min)
+                           .int_timestamp) * 1000
+
+            to_ms = int(arrow.utcnow()
+                        .floor('minute')
+                        .shift(minutes=-self._tf_in_min)
+                        .int_timestamp) * 1000
+
+            # todo: utc date output for starting date
+            self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: "
+                          f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} "
+                          f"till {format_ms_time(to_ms)}", logger.info)
+            needed_pairs = [
+                (p, self._lookback_timeframe) for p in
+                [
+                    s['symbol'] for s in filtered_tickers
+                ] if p not in self._pair_cache
+            ]
+
+            # Get all candles
+            candles = {}
+            if needed_pairs:
+                candles = self._exchange.refresh_latest_ohlcv(
+                    needed_pairs, since_ms=since_ms, cache=False
+                )
+            for i, p in enumerate(filtered_tickers):
+                pair_candles = candles[
+                    (p['symbol'], self._lookback_timeframe)
+                ] if (p['symbol'], self._lookback_timeframe) in candles else None
+                # in case of candle data calculate typical price and quoteVolume for candle
+                if pair_candles is not None and not pair_candles.empty:
+                    pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
+                                                     + pair_candles['close']) / 3
+                    pair_candles['quoteVolume'] = (
+                        pair_candles['volume'] * pair_candles['typical_price']
+                    )
+
+                    # ensure that a rolling sum over the lookback_period is built
+                    # if pair_candles contains more candles than lookback_period
+                    quoteVolume = (pair_candles['quoteVolume']
+                                   .rolling(self._lookback_period)
+                                   .sum()
+                                   .iloc[-1])
+
+                    # replace quoteVolume with range quoteVolume sum calculated above
+                    filtered_tickers[i]['quoteVolume'] = quoteVolume
+                else:
+                    filtered_tickers[i]['quoteVolume'] = 0
+
         if self._min_value > 0:
             filtered_tickers = [
-                    v for v in filtered_tickers if v[self._sort_key] > self._min_value]
+                v for v in filtered_tickers if v[self._sort_key] > self._min_value]
 
         sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key])
 
         # Validate whitelist to only have active market pairs
         pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
-        pairs = self.verify_blacklist(pairs, logger.info)
+        pairs = self.verify_blacklist(pairs, partial(self.log_once, logmethod=logger.info))
         # Limit pairlist to the requested number of pairs
         pairs = pairs[:self._number_pairs]
 

freqtrade/plugins/pairlist/rangestabilityfilter.py

@@ -26,6 +26,7 @@ class RangeStabilityFilter(IPairList):
 
         self._days = pairlistconfig.get('lookback_days', 10)
         self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01)
+        self._max_rate_of_change = pairlistconfig.get('max_rate_of_change', None)
         self._refresh_period = pairlistconfig.get('refresh_period', 1440)
 
         self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
@@ -50,8 +51,12 @@ class RangeStabilityFilter(IPairList):
         """
         Short whitelist method description - used for startup-messages
         """
+        max_rate_desc = ""
+        if self._max_rate_of_change:
+            max_rate_desc = (f" and above {self._max_rate_of_change}")
         return (f"{self.name} - Filtering pairs with rate of change below "
-                f"{self._min_rate_of_change} over the last {plural(self._days, 'day')}.")
+                f"{self._min_rate_of_change}{max_rate_desc} over the "
+                f"last {plural(self._days, 'day')}.")
 
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
@@ -62,10 +67,10 @@ class RangeStabilityFilter(IPairList):
         """
         needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
 
-        since_ms = int(arrow.utcnow()
-                       .floor('day')
-                       .shift(days=-self._days - 1)
-                       .float_timestamp) * 1000
+        since_ms = (arrow.utcnow()
+                         .floor('day')
+                         .shift(days=-self._days - 1)
+                         .int_timestamp) * 1000
         # Get all candles
         candles = {}
         if needed_pairs:
@@ -104,6 +109,17 @@ class RangeStabilityFilter(IPairList):
                               f"which is below the threshold of {self._min_rate_of_change}.",
                               logger.info)
                 result = False
+            if self._max_rate_of_change:
+                if pct_change <= self._max_rate_of_change:
+                    result = True
+                else:
+                    self.log_once(
+                        f"Removed {pair} from whitelist, because rate of change "
+                        f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
+                        f"which is above the threshold of {self._max_rate_of_change}.",
+                        logger.info)
+                    result = False
             self._pair_cache[pair] = result
-
+        else:
+            self.log_once(f"Removed {pair} from whitelist, no candles found.", logger.info)
         return result

freqtrade/plugins/pairlistmanager.py

@@ -28,13 +28,13 @@ class PairListManager():
         self._tickers_needed = False
         for pairlist_handler_config in self._config.get('pairlists', None):
             pairlist_handler = PairListResolver.load_pairlist(
-                    pairlist_handler_config['method'],
-                    exchange=exchange,
-                    pairlistmanager=self,
-                    config=config,
-                    pairlistconfig=pairlist_handler_config,
-                    pairlist_pos=len(self._pairlist_handlers)
-                    )
+                pairlist_handler_config['method'],
+                exchange=exchange,
+                pairlistmanager=self,
+                config=config,
+                pairlistconfig=pairlist_handler_config,
+                pairlist_pos=len(self._pairlist_handlers)
+            )
             self._tickers_needed |= pairlist_handler.needstickers
             self._pairlist_handlers.append(pairlist_handler)
 

freqtrade/plugins/protections/iprotection.py

@@ -25,19 +25,22 @@ class IProtection(LoggingMixin, ABC):
     def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None:
         self._config = config
         self._protection_config = protection_config
+        self._stop_duration_candles: Optional[int] = None
+        self._lookback_period_candles: Optional[int] = None
+
         tf_in_min = timeframe_to_minutes(config['timeframe'])
         if 'stop_duration_candles' in protection_config:
-            self._stop_duration_candles = protection_config.get('stop_duration_candles', 1)
+            self._stop_duration_candles = int(protection_config.get('stop_duration_candles', 1))
             self._stop_duration = (tf_in_min * self._stop_duration_candles)
         else:
             self._stop_duration_candles = None
             self._stop_duration = protection_config.get('stop_duration', 60)
         if 'lookback_period_candles' in protection_config:
-            self._lookback_period_candles = protection_config.get('lookback_period_candles', 1)
+            self._lookback_period_candles = int(protection_config.get('lookback_period_candles', 1))
             self._lookback_period = tf_in_min * self._lookback_period_candles
         else:
             self._lookback_period_candles = None
-            self._lookback_period = protection_config.get('lookback_period', 60)
+            self._lookback_period = int(protection_config.get('lookback_period', 60))
 
         LoggingMixin.__init__(self, logger)
 

freqtrade/plugins/protections/stoploss_guard.py

@@ -54,9 +54,9 @@ class StoplossGuard(IProtection):
 
         trades1 = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until)
         trades = [trade for trade in trades1 if (str(trade.sell_reason) in (
-                    SellType.TRAILING_STOP_LOSS.value, SellType.STOP_LOSS.value,
-                    SellType.STOPLOSS_ON_EXCHANGE.value)
-                      and trade.close_profit and trade.close_profit < 0)]
+            SellType.TRAILING_STOP_LOSS.value, SellType.STOP_LOSS.value,
+            SellType.STOPLOSS_ON_EXCHANGE.value)
+            and trade.close_profit and trade.close_profit < 0)]
 
         if len(trades) < self._trade_limit:
             return False, None, None