Merge pull request #5185 from freqtrade/new_release

New release 2021.6

.devcontainer/Dockerfile

@@ -1,20 +1,21 @@
 FROM freqtradeorg/freqtrade:develop
 
+USER root
 # Install dependencies
 COPY requirements-dev.txt /freqtrade/
+
 RUN apt-get update \
-    && apt-get -y install git mercurial sudo vim \
+    && apt-get -y install git mercurial sudo vim build-essential \
     && apt-get clean \
-    && pip install autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir \
-    && useradd -u 1000 -U -m ftuser \
     && mkdir -p /home/ftuser/.vscode-server /home/ftuser/.vscode-server-insiders /home/ftuser/commandhistory \
     && echo "export PROMPT_COMMAND='history -a'" >> /home/ftuser/.bashrc \
     && echo "export HISTFILE=~/commandhistory/.bash_history" >> /home/ftuser/.bashrc \
-    && mv /root/.local /home/ftuser/.local/ \
     && chown ftuser:ftuser -R /home/ftuser/.local/ \
     && chown ftuser: -R /home/ftuser/
 
 USER ftuser
 
+RUN pip install --user autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir
+
 # Empty the ENTRYPOINT to allow all commands
 ENTRYPOINT []

.dockerignore

@@ -3,6 +3,7 @@
 Dockerfile
 Dockerfile.armhf
 .dockerignore
+docker/
 .coveragerc
 .eggs
 .github

.gitattributes

@@ -1,3 +1,3 @@
-*.py eol=lf
-*.sh eol=lf
-*.ps1 eol=crlf
+*.py    eol=lf
+*.sh    eol=lf
+*.ps1   eol=crlf

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,6 @@
+---
+blank_issues_enabled: false
+contact_links:
+  - name: Discord Server
+    url: https://discord.gg/p7nuUNVfP7
+    about: Ask a question or get community support from our Discord server

.github/workflows/ci.yml

@@ -75,7 +75,7 @@ jobs:
         COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu
       run: |
         # Allow failure for coveralls
-        coveralls -v  || true
+        coveralls || true
 
     - name: Backtesting
       run: |
@@ -301,7 +301,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Cleanup previous runs on this branch
-      uses: rokroskar/workflow-run-cleanup-action@v0.3.2
+      uses: rokroskar/workflow-run-cleanup-action@v0.3.3
       if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/stable' && github.repository == 'freqtrade/freqtrade'"
       env:
         GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
@@ -374,13 +374,6 @@ jobs:
       run: |
         echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
 
-    - name: Build and test and push docker image
-      env:
-        IMAGE_NAME: freqtradeorg/freqtrade
-        BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
-      run: |
-        build_helpers/publish_docker.sh
-
     # We need docker experimental to pull the ARM image.
     - name: Switch docker to experimental
       run: |
@@ -399,12 +392,12 @@ jobs:
     - name: Available platforms
       run: echo ${{ steps.buildx.outputs.platforms }}
 
-    - name: Build Raspberry docker image
+    - name: Build and test and push docker images
       env:
         IMAGE_NAME: freqtradeorg/freqtrade
-        BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}_pi
+        BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
       run: |
-        build_helpers/publish_docker_pi.sh
+        build_helpers/publish_docker_multi.sh
 
 
     - name: Slack Notification

.travis.yml

@@ -46,12 +46,6 @@ jobs:
     - script: mypy freqtrade scripts
       name: mypy
 
-    # - stage: docker
-    #   if: branch in (master, develop, feat/improve_travis) AND (type in (push, cron))
-    #   script:
-    #     - build_helpers/publish_docker.sh
-    #   name: "Build and test and push docker image"
-
 notifications:
   slack:
     secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q=

CONTRIBUTING.md

@@ -12,7 +12,7 @@ Few pointers for contributions:
 - New features need to contain unit tests, must conform to PEP8 (max-line-length = 100) and should be documented with the introduction PR.
 - PR's can be declared as `[WIP]` - which signify Work in Progress Pull Requests (which are not finished).
 
-If you are unsure, discuss the feature on our [discord server](https://discord.gg/MA9v74M), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR.
+If you are unsure, discuss the feature on our [discord server](https://discord.gg/p7nuUNVfP7), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR.
 
 ## Getting started
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9.4-slim-buster as base
+FROM python:3.9.5-slim-buster as base
 
 # Setup env
 ENV LANG C.UTF-8
@@ -10,8 +10,8 @@ ENV FT_APP_ENV="docker"
 
 # Prepare environment
 RUN mkdir /freqtrade \
-  && apt update \
-  && apt install -y sudo \
+  && apt-get update \
+  && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev  \
   && apt-get clean \
   && useradd -u 1000 -G sudo -U -m ftuser \
   && chown ftuser:ftuser /freqtrade \
@@ -22,10 +22,10 @@ WORKDIR /freqtrade
 
 # Install dependencies
 FROM base as python-deps
-RUN apt-get update \
-    && apt-get -y install curl build-essential libssl-dev git \
-    && apt-get clean \
-    && pip install --upgrade pip
+RUN  apt-get update \
+  && apt-get -y install build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \
+  && apt-get clean \
+  && pip install --upgrade pip
 
 # Install TA-lib
 COPY build_helpers/* /tmp/
@@ -49,7 +49,7 @@ USER ftuser
 # Install and execute
 COPY --chown=ftuser:ftuser . /freqtrade/
 
-RUN pip install -e . --user --no-cache-dir \
+RUN pip install -e . --user --no-cache-dir --no-build-isolation \
   && mkdir /freqtrade/user_data/ \
   && freqtrade install-ui
 

README.md

@@ -1,4 +1,4 @@
-# ![freqtrade](docs/assets/freqtrade_poweredby.svg)
+# ![freqtrade](https://raw.githubusercontent.com/freqtrade/freqtrade/develop/docs/assets/freqtrade_poweredby.svg)
 
 [![Freqtrade CI](https://github.com/freqtrade/freqtrade/workflows/Freqtrade%20CI/badge.svg)](https://github.com/freqtrade/freqtrade/actions/)
 [![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop)
@@ -123,7 +123,7 @@ Telegram is not mandatory. However, this is a great way to control your bot. Mor
 - `/stop`: Stops the trader.
 - `/stopbuy`: Stop entering new trades.
 - `/status <trade_id>|[table]`: Lists all or specific open trades.
-- `/profit`: Lists cumulative profit from all finished trades
+- `/profit [<n>]`: Lists cumulative profit from all finished trades, over the last n days.
 - `/forcesell <trade_id>|all`: Instantly sells the given trade (Ignoring `minimum_roi`).
 - `/performance`: Show performance of each finished trade grouped by pair
 - `/balance`: Show account balance per currency.
@@ -145,7 +145,7 @@ The project is currently setup in two main branches:
 
 For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join our slack channel.
 
-Please check out our [discord server](https://discord.gg/MA9v74M).
+Please check out our [discord server](https://discord.gg/p7nuUNVfP7).
 
 You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw).
 
@@ -154,7 +154,7 @@ You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/
 If you discover a bug in the bot, please
 [search our issue tracker](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue)
 first. If it hasn't been reported, please
-[create a new issue](https://github.com/freqtrade/freqtrade/issues/new) and
+[create a new issue](https://github.com/freqtrade/freqtrade/issues/new/choose) and
 ensure you follow the template guide so that our team can assist you as
 quickly as possible.
 
@@ -163,7 +163,7 @@ quickly as possible.
 Have you a great idea to improve the bot you want to share? Please,
 first search if this feature was not [already discussed](https://github.com/freqtrade/freqtrade/labels/enhancement).
 If it hasn't been requested, please
-[create a new request](https://github.com/freqtrade/freqtrade/issues/new)
+[create a new request](https://github.com/freqtrade/freqtrade/issues/new/choose)
 and ensure you follow the template guide so that it does not get lost
 in the bug reports.
 
@@ -178,7 +178,7 @@ to understand the requirements before sending your pull-requests.
 Coding is not a necessity to contribute - maybe start with improving our documentation?
 Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/good%20first%20issue) can be good first contributions, and will help get you familiar with the codebase.
 
-**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/MA9v74M) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it.
+**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/p7nuUNVfP7) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it.
 
 **Important:** Always create your PR against the `develop` branch, not `stable`.
 

build_helpers/install_ta-lib.sh

@@ -8,10 +8,13 @@ if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then
   tar zxvf ta-lib-0.4.0-src.tar.gz
   cd ta-lib \
   && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \
+  && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' -o config.guess \
+  && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub \
   && ./configure --prefix=${INSTALL_LOC}/ \
-  && make \
+  && make -j$(nproc) \
   && which sudo && sudo make install || make install \
   && cd ..
 else
   echo "TA-lib already installed, skipping installation"
 fi
+#  && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \

build_helpers/install_windows.ps1

@@ -1,16 +1,15 @@
 # Downloads don't work automatically, since the URL is regenerated via javascript.
 # Downloaded from https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib
-# Invoke-WebRequest -Uri "https://download.lfd.uci.edu/pythonlibs/xxxxxxx/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl" -OutFile "TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl"
 
 python -m pip install --upgrade pip
 
 $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"
 
 if ($pyv -eq '3.7') {
-    pip install build_helpers\TA_Lib-0.4.19-cp37-cp37m-win_amd64.whl
+    pip install build_helpers\TA_Lib-0.4.20-cp37-cp37m-win_amd64.whl
 }
 if ($pyv -eq '3.8') {
-    pip install build_helpers\TA_Lib-0.4.19-cp38-cp38-win_amd64.whl
+    pip install build_helpers\TA_Lib-0.4.20-cp38-cp38-win_amd64.whl
 }
 
 pip install -r requirements-dev.txt

build_helpers/publish_docker_multi.sh

@@ -1,21 +1,48 @@
 #!/bin/sh
 
+# The below assumes a correctly setup docker buildx environment
+
 # Replace / with _ to create a valid tag
 TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
 TAG_PLOT=${TAG}_plot
+TAG_PI="${TAG}_pi"
+
+PI_PLATFORM="linux/arm/v7"
 echo "Running for ${TAG}"
+CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache
 
 # Add commit and commit_message to docker container
 echo "${GITHUB_SHA}" > freqtrade_commit
 
 if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then
     echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache"
+    # Build regular image
     docker build -t freqtrade:${TAG} .
+    # Build PI image
+    docker buildx build \
+        --cache-to=type=registry,ref=${CACHE_TAG} \
+        -f docker/Dockerfile.armhf \
+        --platform ${PI_PLATFORM} \
+        -t ${IMAGE_NAME}:${TAG_PI} --push .
 else
     echo "event ${GITHUB_EVENT_NAME}: building with cache"
-    # Pull last build to avoid rebuilding the whole image
+    # Build regular image
     docker pull ${IMAGE_NAME}:${TAG}
     docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} .
+
+    # Pull last build to avoid rebuilding the whole image
+    # docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG}
+    docker buildx build \
+        --cache-from=type=registry,ref=${CACHE_TAG} \
+        --cache-to=type=registry,ref=${CACHE_TAG} \
+        -f docker/Dockerfile.armhf \
+        --platform ${PI_PLATFORM} \
+        -t ${IMAGE_NAME}:${TAG_PI} --push .
+fi
+
+if [ $? -ne 0 ]; then
+    echo "failed building multiarch images"
+    return 1
 fi
 # Tag image for upload and next build step
 docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG
@@ -24,11 +51,6 @@ docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t fre
 
 docker tag freqtrade:$TAG_PLOT ${IMAGE_NAME}:$TAG_PLOT
 
-if [ $? -ne 0 ]; then
-    echo "failed building image"
-    return 1
-fi
-
 # Run backtest
 docker run --rm -v $(pwd)/config_bittrex.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy
 
@@ -37,23 +59,29 @@ if [ $? -ne 0 ]; then
     return 1
 fi
 
-if [ $? -ne 0 ]; then
-    echo "failed tagging image"
-    return 1
-fi
-
-# Tag as latest for develop builds
-if [ "${TAG}" = "develop" ]; then
-    docker tag freqtrade:$TAG ${IMAGE_NAME}:latest
-fi
-
-# Show all available images
 docker images
 
 docker push ${IMAGE_NAME}
 docker push ${IMAGE_NAME}:$TAG_PLOT
 docker push ${IMAGE_NAME}:$TAG
+
+# Create multiarch image
+# Make sure that all images contained here are pushed to github first.
+# Otherwise installation might fail.
+
+docker manifest create freqtradeorg/freqtrade:${TAG} ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI}
+docker manifest push freqtradeorg/freqtrade:${TAG}
+
+# Tag as latest for develop builds
+if [ "${TAG}" = "develop" ]; then
+    docker manifest create freqtradeorg/freqtrade:latest ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI}
+    docker manifest push freqtradeorg/freqtrade:latest
+fi
+
+
+docker images
+
 if [ $? -ne 0 ]; then
-    echo "failed pushing repo"
+    echo "failed building image"
     return 1
 fi

build_helpers/publish_docker_pi.sh

@@ -1,36 +0,0 @@
-#!/bin/sh
-
-# The below assumes a correctly setup docker buildx environment
-
-# Replace / with _ to create a valid tag
-TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
-PI_PLATFORM="linux/arm/v7"
-echo "Running for ${TAG}"
-CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache
-
-# Add commit and commit_message to docker container
-echo "${GITHUB_SHA}" > freqtrade_commit
-
-if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then
-    echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache"
-    docker buildx build \
-        --cache-to=type=registry,ref=${CACHE_TAG} \
-        -f Dockerfile.armhf \
-        --platform ${PI_PLATFORM} \
-        -t ${IMAGE_NAME}:${TAG} --push .
-else
-    echo "event ${GITHUB_EVENT_NAME}: building with cache"
-    # Pull last build to avoid rebuilding the whole image
-    # docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG}
-    docker buildx build \
-        --cache-from=type=registry,ref=${CACHE_TAG} \
-        --cache-to=type=registry,ref=${CACHE_TAG} \
-        -f Dockerfile.armhf \
-        --platform ${PI_PLATFORM} \
-        -t ${IMAGE_NAME}:${TAG} --push .
-fi
-
-if [ $? -ne 0 ]; then
-    echo "failed building image"
-    return 1
-fi

config_full.json.example

@@ -23,7 +23,8 @@
     "stoploss": -0.10,
     "unfilledtimeout": {
         "buy": 10,
-        "sell": 30
+        "sell": 30,
+        "unit": "minutes"
     },
     "bid_strategy": {
         "price_side": "bid",
@@ -164,11 +165,22 @@
             "startup": "on",
             "buy": "on",
             "buy_fill": "on",
-            "sell": "on",
+            "sell": {
+                "roi": "off",
+                "emergency_sell": "off",
+                "force_sell": "off",
+                "sell_signal": "off",
+                "trailing_stop_loss": "off",
+                "stop_loss": "off",
+                "stoploss_on_exchange": "off",
+                "custom_sell": "off"
+            },
             "sell_fill": "on",
             "buy_cancel": "on",
             "sell_cancel": "on"
-        }
+        },
+        "reload": true,
+        "balance_dust_level": 0.01
     },
     "api_server": {
         "enabled": false,

docker/Dockerfile.armhf

@@ -1,4 +1,4 @@
-FROM --platform=linux/arm/v7 python:3.7.10-slim-buster as base
+FROM python:3.7.10-slim-buster as base
 
 # Setup env
 ENV LANG C.UTF-8
@@ -11,7 +11,7 @@ ENV FT_APP_ENV="docker"
 # Prepare environment
 RUN mkdir /freqtrade \
   && apt-get update \
-  && apt-get -y install libatlas3-base curl sqlite3 libhdf5-serial-dev sudo \
+  && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-dev \
   && apt-get clean \
   && useradd -u 1000 -G sudo -U -m ftuser \
   && chown ftuser:ftuser /freqtrade \
@@ -22,7 +22,8 @@ WORKDIR /freqtrade
 
 # Install dependencies
 FROM base as python-deps
-RUN  apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 \
+RUN  apt-get update \
+  && apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 pkg-config cmake gcc \
   && apt-get clean \
   && pip install --upgrade pip \
   && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf
@@ -49,7 +50,7 @@ USER ftuser
 # Install and execute
 COPY --chown=ftuser:ftuser . /freqtrade/
 
-RUN pip install -e . --user --no-cache-dir \
+RUN pip install -e . --user --no-cache-dir --no-build-isolation\
   && mkdir /freqtrade/user_data/ \
   && freqtrade install-ui
 

docs/advanced-hyperopt.md

@@ -289,7 +289,7 @@ Given the following result from hyperopt:
 ```
 Best result:
 
-    44/100:    135 trades. Avg profit  0.57%. Total profit  0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367
+    44/100:    135 trades. Avg profit  0.57%. Total profit  0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367
 
 Buy hyperspace params:
 {    'adx-value': 44,

docs/backtesting.md

@@ -19,7 +19,7 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                              [--enable-protections]
                              [--dry-run-wallet DRY_RUN_WALLET]
                              [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
-                             [--export EXPORT] [--export-filename PATH]
+                             [--export {none,trades}] [--export-filename PATH]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -63,8 +63,8 @@ optional arguments:
                         name is injected into the filename (so `backtest-
                         data.json` becomes `backtest-data-
                         DefaultStrategy.json`
-  --export EXPORT       Export backtest results, argument are: trades.
-                        Example: `--export=trades`
+  --export {none,trades}
+                        Export backtest results (default: trades).
   --export-filename PATH
                         Save backtest results to the file with this filename.
                         Requires `--export` to be set as well. Example:
@@ -100,7 +100,7 @@ Strategy arguments:
 Now you have good Buy and Sell strategies and some historic data, you want to test it against
 real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting).
 
-Backtesting will use the crypto-currencies (pairs) from your config file and load historical candle (OHCLV) data from `user_data/data/<exchange>` by default.
+Backtesting will use the crypto-currencies (pairs) from your config file and load historical candle (OHLCV) data from `user_data/data/<exchange>` by default.
 If no data is available for the exchange / pair / timeframe combination, backtesting will ask you to download them first using `freqtrade download-data`.
 For details on downloading, please refer to the [Data Downloading](data-download.md) section in the documentation.
 
@@ -110,11 +110,16 @@ All profit calculations include fees, and freqtrade will use the exchange's defa
 
 !!! Warning "Using dynamic pairlists for backtesting"
     Using dynamic pairlists is possible, however it relies on the current market conditions - which will not reflect the historic status of the pairlist.
-    Also, when using pairlists other than StaticPairlist, reproducability of backtesting-results cannot be guaranteed.
+    Also, when using pairlists other than StaticPairlist, reproducibility of backtesting-results cannot be guaranteed.
     Please read the [pairlists documentation](plugins.md#pairlists) for more information.
 
     To achieve reproducible results, best generate a pairlist via the [`test-pairlist`](utils.md#test-pairlist) command and use that as static pairlist.
 
+!!! Note
+    By default, Freqtrade will export backtesting results to `user_data/backtest_results`.
+    The exported trades can be used for [further analysis](#further-backtest-result-analysis) or can be used by the [plotting sub-command](plotting.md#plot-price-and-indicators) (`freqtrade plot-dataframe`) in the scripts directory.
+
+
 ### Starting balance
 
 Backtesting will require a starting balance, which can be provided as `--dry-run-wallet <balance>` or `--starting-balance <balance>` command line argument, or via `dry_run_wallet` configuration setting.
@@ -174,13 +179,13 @@ Where `SampleStrategy1` and `AwesomeStrategy` refer to class names of strategies
 
 ---
 
-Exporting trades to file
+Prevent exporting trades to file
 
 ```bash
-freqtrade backtesting --strategy backtesting --export trades --config config.json 
+freqtrade backtesting --strategy backtesting --export none --config config.json 
 ```
 
-The exported trades can be used for [further analysis](#further-backtest-result-analysis), or can be used by the plotting script `plot_dataframe.py` in the scripts directory.
+Only use this if you're sure you'll not want to plot or analyze your results further.
 
 ---
 
@@ -237,29 +242,29 @@ The most important in the backtesting is to understand the result.
 A backtesting result will look like that:
 
 ```
-========================================================= BACKTESTING REPORT ========================================================
-| Pair     |   Buys |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Wins |  Draws |  Losses |
-|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|--------:|
-| ADA/BTC  |     35 |          -0.11 |          -3.88 |      -0.00019428 |          -1.94 | 4:35:00        |    14 |      0 |      21 |
-| ARK/BTC  |     11 |          -0.41 |          -4.52 |      -0.00022647 |          -2.26 | 2:03:00        |     3 |      0 |       8 |
-| BTS/BTC  |     32 |           0.31 |           9.78 |       0.00048938 |           4.89 | 5:05:00        |    18 |      0 |      14 |
-| DASH/BTC |     13 |          -0.08 |          -1.07 |      -0.00005343 |          -0.53 | 4:39:00        |     6 |      0 |       7 |
-| ENG/BTC  |     18 |           1.36 |          24.54 |       0.00122807 |          12.27 | 2:50:00        |     8 |      0 |      10 |
-| EOS/BTC  |     36 |           0.08 |           3.06 |       0.00015304 |           1.53 | 3:34:00        |    16 |      0 |      20 |
-| ETC/BTC  |     26 |           0.37 |           9.51 |       0.00047576 |           4.75 | 6:14:00        |    11 |      0 |      15 |
-| ETH/BTC  |     33 |           0.30 |           9.96 |       0.00049856 |           4.98 | 7:31:00        |    16 |      0 |      17 |
-| IOTA/BTC |     32 |           0.03 |           1.09 |       0.00005444 |           0.54 | 3:12:00        |    14 |      0 |      18 |
-| LSK/BTC  |     15 |           1.75 |          26.26 |       0.00131413 |          13.13 | 2:58:00        |     6 |      0 |       9 |
-| LTC/BTC  |     32 |          -0.04 |          -1.38 |      -0.00006886 |          -0.69 | 4:49:00        |    11 |      0 |      21 |
-| NANO/BTC |     17 |           1.26 |          21.39 |       0.00107058 |          10.70 | 1:55:00        |    10 |      0 |       7 |
-| NEO/BTC  |     23 |           0.82 |          18.97 |       0.00094936 |           9.48 | 2:59:00        |    10 |      0 |      13 |
-| REQ/BTC  |      9 |           1.17 |          10.54 |       0.00052734 |           5.27 | 3:47:00        |     4 |      0 |       5 |
-| XLM/BTC  |     16 |           1.22 |          19.54 |       0.00097800 |           9.77 | 3:15:00        |     7 |      0 |       9 |
-| XMR/BTC  |     23 |          -0.18 |          -4.13 |      -0.00020696 |          -2.07 | 5:30:00        |    12 |      0 |      11 |
-| XRP/BTC  |     35 |           0.66 |          22.96 |       0.00114897 |          11.48 | 3:49:00        |    12 |      0 |      23 |
-| ZEC/BTC  |     22 |          -0.46 |         -10.18 |      -0.00050971 |          -5.09 | 2:22:00        |     7 |      0 |      15 |
-| TOTAL    |    429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00        |   186 |      0 |     243 |
-========================================================= SELL REASON STATS =========================================================
+========================================================= BACKTESTING REPORT ==========================================================
+| Pair     |   Buys |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration |  Wins Draws Loss   Win%  |
+|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:-------------|-------------------------:|
+| ADA/BTC  |     35 |          -0.11 |          -3.88 |      -0.00019428 |          -1.94 | 4:35:00      |    14     0    21   40.0 |
+| ARK/BTC  |     11 |          -0.41 |          -4.52 |      -0.00022647 |          -2.26 | 2:03:00      |     3     0     8   27.3 |
+| BTS/BTC  |     32 |           0.31 |           9.78 |       0.00048938 |           4.89 | 5:05:00      |    18     0    14   56.2 |
+| DASH/BTC |     13 |          -0.08 |          -1.07 |      -0.00005343 |          -0.53 | 4:39:00      |     6     0     7   46.2 |
+| ENG/BTC  |     18 |           1.36 |          24.54 |       0.00122807 |          12.27 | 2:50:00      |     8     0    10   44.4 |
+| EOS/BTC  |     36 |           0.08 |           3.06 |       0.00015304 |           1.53 | 3:34:00      |    16     0    20   44.4 |
+| ETC/BTC  |     26 |           0.37 |           9.51 |       0.00047576 |           4.75 | 6:14:00      |    11     0    15   42.3 |
+| ETH/BTC  |     33 |           0.30 |           9.96 |       0.00049856 |           4.98 | 7:31:00      |    16     0    17   48.5 |
+| IOTA/BTC |     32 |           0.03 |           1.09 |       0.00005444 |           0.54 | 3:12:00      |    14     0    18   43.8 |
+| LSK/BTC  |     15 |           1.75 |          26.26 |       0.00131413 |          13.13 | 2:58:00      |     6     0     9   40.0 |
+| LTC/BTC  |     32 |          -0.04 |          -1.38 |      -0.00006886 |          -0.69 | 4:49:00      |    11     0    21   34.4 |
+| NANO/BTC |     17 |           1.26 |          21.39 |       0.00107058 |          10.70 | 1:55:00      |    10     0     7   58.5 |
+| NEO/BTC  |     23 |           0.82 |          18.97 |       0.00094936 |           9.48 | 2:59:00      |    10     0    13   43.5 |
+| REQ/BTC  |      9 |           1.17 |          10.54 |       0.00052734 |           5.27 | 3:47:00      |     4     0     5   44.4 |
+| XLM/BTC  |     16 |           1.22 |          19.54 |       0.00097800 |           9.77 | 3:15:00      |     7     0     9   43.8 |
+| XMR/BTC  |     23 |          -0.18 |          -4.13 |      -0.00020696 |          -2.07 | 5:30:00      |    12     0    11   52.2 |
+| XRP/BTC  |     35 |           0.66 |          22.96 |       0.00114897 |          11.48 | 3:49:00      |    12     0    23   34.3 |
+| ZEC/BTC  |     22 |          -0.46 |         -10.18 |      -0.00050971 |          -5.09 | 2:22:00      |     7     0    15   31.8 |
+| TOTAL    |    429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00      |   186     0   243   43.4 |
+========================================================= SELL REASON STATS ==========================================================
 | Sell Reason        |   Sells |  Wins |  Draws |  Losses |
 |:-------------------|--------:|------:|-------:|--------:|
 | trailing_stop_loss |     205 |   150 |      0 |      55 |
@@ -267,11 +272,11 @@ A backtesting result will look like that:
 | sell_signal        |      56 |    36 |      0 |      20 |
 | force_sell         |       2 |     0 |      0 |       2 |
 ====================================================== LEFT OPEN TRADES REPORT ======================================================
-| Pair     |   Buys |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Wins |  Draws |  Losses |
-|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|--------:|
-| ADA/BTC  |      1 |           0.89 |           0.89 |       0.00004434 |           0.44 | 6:00:00        |     1 |      0 |       0 |
-| LTC/BTC  |      1 |           0.68 |           0.68 |       0.00003421 |           0.34 | 2:00:00        |     1 |      0 |       0 |
-| TOTAL    |      2 |           0.78 |           1.57 |       0.00007855 |           0.78 | 4:00:00        |     2 |      0 |       0 |
+| Pair     |   Buys |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Win Draw Loss Win% |
+|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|--------------------:|
+| ADA/BTC  |      1 |           0.89 |           0.89 |       0.00004434 |           0.44 | 6:00:00        |    1    0    0  100 |
+| LTC/BTC  |      1 |           0.68 |           0.68 |       0.00003421 |           0.34 | 2:00:00        |    1    0    0  100 |
+| TOTAL    |      2 |           0.78 |           1.57 |       0.00007855 |           0.78 | 4:00:00        |    2    0    0  100 |
 =============== SUMMARY METRICS ===============
 | Metric                | Value               |
 |-----------------------+---------------------|
@@ -279,7 +284,7 @@ A backtesting result will look like that:
 | Backtesting to        | 2019-05-01 00:00:00 |
 | Max open trades       | 3                   |
 |                       |                     |
-| Total trades          | 429                 |
+| Total/Daily Avg Trades| 429 / 3.575         |
 | Starting balance      | 0.01000000 BTC      |
 | Final balance         | 0.01762792 BTC      |
 | Absolute profit       | 0.00762792 BTC      |
@@ -297,6 +302,8 @@ A backtesting result will look like that:
 | Days win/draw/lose    | 12 / 82 / 25        |
 | Avg. Duration Winners | 4:23:00             |
 | Avg. Duration Loser   | 6:55:00             |
+| Zero Duration Trades  | 4.6% (20)           |
+| Rejected Buy signals  | 3089                |
 |                       |                     |
 | Min balance           | 0.00945123 BTC      |
 | Max balance           | 0.01846651 BTC      |
@@ -318,7 +325,7 @@ The last line will give you the overall performance of your strategy,
 here:
 
 ```
-| TOTAL    |         429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00        |      186 |    243 |
+| TOTAL    |    429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00      |   186     0   243   43.4 |
 ```
 
 The bot has made `429` trades for an average duration of `4:12:00`, with a performance of `76.20%` (profit), that means it has
@@ -366,12 +373,11 @@ It contains some useful key metrics about performance of your strategy on backte
 | Backtesting to        | 2019-05-01 00:00:00 |
 | Max open trades       | 3                   |
 |                       |                     |
-| Total trades          | 429                 |
+| Total/Daily Avg Trades| 429 / 3.575         |
 | Starting balance      | 0.01000000 BTC      |
 | Final balance         | 0.01762792 BTC      |
 | Absolute profit       | 0.00762792 BTC      |
 | Total profit %        | 76.2%               |
-| Trades per day        | 3.575               |
 | Avg. stake amount     | 0.001      BTC      |
 | Total trade volume    | 0.429      BTC      |
 |                       |                     |
@@ -384,6 +390,8 @@ It contains some useful key metrics about performance of your strategy on backte
 | Days win/draw/lose    | 12 / 82 / 25        |
 | Avg. Duration Winners | 4:23:00             |
 | Avg. Duration Loser   | 6:55:00             |
+| Zero Duration Trades  | 4.6% (20)           |
+| Rejected Buy signals  | 3089                |
 |                       |                     |
 | Min balance           | 0.00945123 BTC      |
 | Max balance           | 0.01846651 BTC      |
@@ -400,12 +408,11 @@ It contains some useful key metrics about performance of your strategy on backte
 
 - `Backtesting from` / `Backtesting to`: Backtesting range (usually defined with the `--timerange` option).
 - `Max open trades`: Setting of `max_open_trades` (or `--max-open-trades`) - or number of pairs in the pairlist (whatever is lower).
-- `Total trades`: Identical to the total trades of the backtest output table.
+- `Total/Daily Avg Trades`: Identical to the total trades of the backtest output table / Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy).
 - `Starting balance`: Start balance - as given by dry-run-wallet (config or command line).
 - `Final balance`: Final balance - starting balance + absolute profit.
 - `Absolute profit`: Profit made in stake currency.
 - `Total profit %`: Total profit. Aligned to the `TOTAL` row's `Tot Profit %` from the first table. Calculated as `(End capital − Starting capital) / Starting capital`.
-- `Trades per day`: Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy).
 - `Avg. stake amount`: Average stake amount, either `stake_amount` or the average when using dynamic stake amount.
 - `Total trade volume`: Volume generated on the exchange to reach the above profit.
 - `Best Pair` / `Worst Pair`: Best and worst performing pair, and it's corresponding `Cum Profit %`.
@@ -413,6 +420,8 @@ It contains some useful key metrics about performance of your strategy on backte
 - `Best day` / `Worst day`: Best and worst day based on daily profit.
 - `Days win/draw/lose`: Winning / Losing days (draws are usually days without closed trade).
 - `Avg. Duration Winners` / `Avg. Duration Loser`: Average durations for winning and losing trades.
+- `Zero Duration Trades`: A number of trades that completed within same candle as they opened and had `trailing_stop_loss` sell reason. A significant amount of such trades may indicate that strategy is exploiting trailing stoploss behavior in backtesting and produces unrealistic results.
+- `Rejected Buy signals`: Buy signals that could not be acted upon due to max_open_trades being reached.
 - `Min balance` / `Max balance`: Lowest and Highest Wallet balance during the backtest period.
 - `Drawdown`: Maximum drawdown experienced. For example, the value of 50% means that from highest to subsequent lowest point, a 50% drop was experienced).
 - `Drawdown high` / `Drawdown low`: Profit at the beginning and end of the largest drawdown period. A negative low value means initial capital lost.
@@ -435,6 +444,7 @@ Since backtesting lacks some detailed information about what happens within a ca
 - Stoploss is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` sell reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes
 - Low happens before high for stoploss, protecting capital first
 - Trailing stoploss
+  - Trailing Stoploss is only adjusted if it's below the candle's low (otherwise it would be triggered)
   - High happens first - adjusting stoploss
   - Low uses the adjusted stoploss (so sells with large high-low difference are backtested correctly)
   - ROI applies before trailing-stop, ensuring profits are "top-capped" at ROI if both ROI and trailing stop applies
@@ -472,11 +482,11 @@ There will be an additional table comparing win/losses of the different strategi
 Detailed output for all strategies one after the other will be available, so make sure to scroll up to see the details per strategy.
 
 ```
-=========================================================== STRATEGY SUMMARY ===========================================================
-| Strategy    |   Buys |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Wins |  Draws | Losses |
-|:------------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|-------:|
-| Strategy1   |    429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00        |   186 |      0 |    243 |
-| Strategy2   |   1487 |          -0.13 |        -197.58 |      -0.00988917 |         -98.79 | 4:43:00        |   662 |      0 |    825 |
+=========================================================== STRATEGY SUMMARY =========================================================================
+| Strategy    |   Buys |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Wins |  Draws | Losses | Drawdown % |
+|:------------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|-------:|-----------:|
+| Strategy1   |    429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00        |   186 |      0 |    243 |       45.2 |
+| Strategy2   |   1487 |          -0.13 |        -197.58 |      -0.00988917 |         -98.79 | 4:43:00        |   662 |      0 |    825 |     241.68 |
 ```
 
 ## Next step

docs/configuration.md

@@ -68,8 +68,9 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md#trailing-stop-loss-only-once-the-trade-has-reached-a-certain-offset). [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `0.0` (no offset).* <br> **Datatype:** Float
 | `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.*  <br> **Datatype:** Boolean
 | `fee` | Fee used during backtesting / dry-runs. Should normally not be configured, which has freqtrade fall back to the exchange default fee. Set as ratio (e.g. 0.001 = 0.1%). Fee is applied twice for each trade, once when buying, once when selling. <br> **Datatype:** Float (as ratio)
-| `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
-| `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
+| `unfilledtimeout.buy` | **Required.** How long (in minutes or seconds) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
+| `unfilledtimeout.sell` | **Required.** How long (in minutes or seconds) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
+| `unfilledtimeout.unit` | Unit to use in unfilledtimeout setting. Note: If you set unfilledtimeout.unit to "seconds", "internals.process_throttle_secs" must be inferior or equal to timeout [Strategy Override](#parameters-in-the-strategy). <br> *Defaults to `minutes`.* <br> **Datatype:** String
 | `bid_strategy.price_side` | Select the side of the spread the bot should look at to get the buy rate. [More information below](#buy-price-side).<br> *Defaults to `bid`.* <br> **Datatype:** String (either `ask` or `bid`).
 | `bid_strategy.ask_last_balance` | **Required.** Interpolate the bidding price. More information [below](#buy-price-without-orderbook-enabled).
 | `bid_strategy.use_order_book` | Enable buying using the rates in [Order Book Bids](#buy-price-with-orderbook-enabled). <br> **Datatype:** Boolean
@@ -101,10 +102,11 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded. <br>*Defaults to `60` minutes.* <br> **Datatype:** Positive Integer
 | `exchange.skip_pair_validation` | Skip pairlist validation on startup.<br>*Defaults to `false`<br> **Datatype:** Boolean
 | `exchange.skip_open_order_update` | Skips open order updates on startup should the exchange cause problems. Only relevant in live conditions.<br>*Defaults to `false`<br> **Datatype:** Boolean
+| `exchange.log_responses` | Log relevant exchange responses. For debug mode only - use with care.<br>*Defaults to `false`<br> **Datatype:** Boolean
 | `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation.
 | `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now. <br>*Defaults to `true`.* <br> **Datatype:** Boolean
 | `pairlists` | Define one or more pairlists to be used. [More information](plugins.md#pairlists-and-pairlist-handlers). <br>*Defaults to `StaticPairList`.*  <br> **Datatype:** List of Dicts
-| `protections` | Define one or more protections to be used. [More information](plugins.md#protections). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** List of Dicts
+| `protections` | Define one or more protections to be used. [More information](plugins.md#protections). <br> **Datatype:** List of Dicts
 | `telegram.enabled` | Enable the usage of Telegram. <br> **Datatype:** Boolean
 | `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`. <br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
 | `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. <br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
@@ -139,7 +141,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 
 ### Parameters in the strategy
 
-The following parameters can be set in either configuration file or strategy.
+The following parameters can be set in configuration file or strategy.
 Values set in the configuration file always overwrite values set in the strategy.
 
 * `minimal_roi`
@@ -155,7 +157,6 @@ Values set in the configuration file always overwrite values set in the strategy
 * `order_time_in_force`
 * `unfilledtimeout`
 * `disable_dataframe_checks`
-* `protections`
 * `use_sell_signal` (ask_strategy)
 * `sell_profit_only` (ask_strategy)
 * `sell_profit_offset` (ask_strategy)
@@ -169,7 +170,7 @@ There are several methods to configure how much of the stake currency the bot wi
 #### Minimum trade stake
 
 The minimum stake amount will depend by exchange and pair, and is usually listed in the exchange support pages.
-Assuming the minimum tradable amount for XRP/USD is 20 XRP (given by the exchange), and the price is 0.4$.
+Assuming the minimum tradable amount for XRP/USD is 20 XRP (given by the exchange), and the price is 0.6$.
 
 The minimum stake amount to buy this pair is therefore `20 * 0.6 ~= 12`.
 This exchange has also a limit on USD - where all orders must be > 10$ - which however does not apply in this case.
@@ -303,6 +304,9 @@ For example, if your strategy is using a 1h timeframe, and you only want to buy
   },
 ```
 
+!!! Note
+    This setting resets with each new candle, so it will not prevent sticking-signals from executing on the 2nd or 3rd candle they're active. Best use a "trigger" selector for buy signals, which are only active for one candle.
+
 ### Understand order_types
 
 The `order_types` configuration parameter maps actions (`buy`, `sell`, `stoploss`, `emergencysell`, `forcesell`, `forcebuy`) to order-types (`market`, `limit`, ...) as well as configures stoploss to be on the exchange and defines stoploss on exchange update interval in seconds.
@@ -402,8 +406,8 @@ The possible values are: `gtc` (default), `fok` or `ioc`.
 ```
 
 !!! Warning
-    This is an ongoing work. For now it is supported only for binance and only for buy orders.
-    Please don't change the default value unless you know what you are doing.
+    This is an ongoing work. For now it is supported only for binance.
+    Please don't change the default value unless you know what you are doing and have researched the impact of using different values.
 
 ### Exchange configuration
 
@@ -499,7 +503,8 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo
 * API-keys may or may not be provided. Only Read-Only operations (i.e. operations that do not alter account state) on the exchange are performed in dry-run mode.
 * Wallets (`/balance`) are simulated based on `dry_run_wallet`.
 * Orders are simulated, and will not be posted to the exchange.
-* Orders are assumed to fill immediately, and will never time out.
+* Market orders fill based on orderbook volume the moment the order is placed.
+* Limit orders fill once price reaches the defined level - or time out based on `unfilledtimeout` settings.
 * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled.
 * Open orders (not trades, which are stored in the database) are reset on bot restart.
 

docs/developer.md

@@ -2,7 +2,7 @@
 
 This page is intended for developers of Freqtrade, people who want to contribute to the Freqtrade codebase or documentation, or people who want to understand the source code of the application they're running.
 
-All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/MA9v74M) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) where you can ask questions.
+All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/p7nuUNVfP7) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) where you can ask questions.
 
 ## Documentation
 

docs/docker_quickstart.md

@@ -10,7 +10,7 @@ Start by downloading and installing Docker CE for your platform:
 * [Windows](https://docs.docker.com/docker-for-windows/install/)
 * [Linux](https://docs.docker.com/install/)
 
-To simplify running freqtrade, please install [`docker-compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the below [docker quick start guide](#docker-quick-start).
+To simplify running freqtrade, [`docker-compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the below [docker quick start guide](#docker-quick-start).
 
 ## Freqtrade with docker-compose
 
@@ -48,6 +48,8 @@ Create a new directory and place the [docker-compose file](https://raw.githubuse
     # Download the docker-compose file from the repository
     curl https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml -o docker-compose.yml
 
+    # Edit the compose file to use an image named `*_pi` (stable_pi or develop_pi)
+
     # Pull the freqtrade image
     docker-compose pull
 
@@ -65,6 +67,40 @@ Create a new directory and place the [docker-compose file](https://raw.githubuse
         # image: freqtradeorg/freqtrade:develop_pi
         ```
 
+=== "ARM 64 Systenms (Mac M1, Raspberry Pi 4, Jetson Nano)"
+    In case of a Mac M1, make sure that your docker installation is running in native mode
+    Arm64 images are not yet provided via Docker Hub and need to be build locally first.
+    Depending on the device, this may take a few minutes (Apple M1) or multiple hours (Raspberry Pi)
+
+    ``` bash
+    # Clone Freqtrade repository
+    git clone  https://github.com/freqtrade/freqtrade.git
+    cd freqtrade
+    # Optionally switch to the stable version
+    git checkout stable   
+
+    # Modify your docker-compose file to enable building and change the image name 
+    # (see the Note Box below for necessary changes)
+
+    # Build image
+    docker-compose build
+    
+    # Create user directory structure
+    docker-compose run --rm freqtrade create-userdir --userdir user_data
+
+    # Create configuration - Requires answering interactive questions
+    docker-compose run --rm freqtrade new-config --config user_data/config.json
+    ```
+
+    !!! Note "Change your docker Image"
+        You have to change the docker image in the docker-compose file for your arm64 build to work properly.
+        ``` yml
+        image: freqtradeorg/freqtrade:custom_arm64
+        build:
+          context: .
+          dockerfile: "Dockerfile"
+        ```
+
 The above snippet creates a new directory called `ft_userdata`, downloads the latest compose file and pulls the freqtrade image.
 The last 2 steps in the snippet create the directory with `user_data`, as well as (interactively) the default configuration based on your selections.
 

docs/exchanges.md

@@ -14,11 +14,10 @@ Accounts having BNB accounts use this to pay for fees - if your first trade happ
 
 ### Binance sites
 
-Binance has been split into 3, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized.
+Binance has been split into 2, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized.
 
 * [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance`.
 * [binance.us](https://www.binance.us/) - US based users. Use exchange id: `binanceus`.
-* [binance.je](https://www.binance.je/) - Binance Jersey, trading fiat currencies. Use exchange id: `binanceje`.
 
 ## Kraken
 
@@ -54,6 +53,9 @@ Due to the heavy rate-limiting applied by Kraken, the following configuration se
 
 Bittrex does not support market orders. If you have a message at the bot startup about this, you should change order type values set in your configuration and/or in the strategy from `"market"` to `"limit"`. See some more details on this [here in the FAQ](faq.md#im-getting-the-exchange-bittrex-does-not-support-market-orders-message-and-cannot-run-my-strategy).
 
+Bittrex also does not support `VolumePairlist` due to limited / split API constellation at the moment.
+Please use `StaticPairlist`. Other pairlists (other than `VolumePairlist`) should not be affected.
+
 ### Restricted markets
 
 Bittrex split its exchange into US and International versions.

docs/faq.md

@@ -136,6 +136,22 @@ On Windows, the `--logfile` option is also supported by Freqtrade and you can us
 > type \path\to\mylogfile.log | findstr "something"
 ```
 
+### Why does freqtrade not have GPU support?
+
+First of all, most indicator libraries don't have GPU support - as such, there would be little benefit for indicator calculations.
+The GPU improvements would only apply to pandas-native calculations - or ones written by yourself.
+
+For hyperopt, freqtrade is using scikit-optimize, which is built on top of scikit-learn.
+Their statement about GPU support is [pretty clear](https://scikit-learn.org/stable/faq.html#will-you-add-gpu-support).
+
+GPU's also are only good at crunching numbers (floating point operations). 
+For hyperopt, we need both number-crunching (find next parameters) and running python code (running backtesting). 
+As such, GPU's are not too well suited for most parts of hyperopt.
+
+The benefit of using GPU would therefore be pretty slim - and will not justify the complexity introduced by trying to add GPU support.
+
+There is however nothing preventing you from using GPU-enabled indicators within your strategy if you think you must have this - you will however probably be disappointed by the slim gain that will give you (compared to the complexity).
+
 ## Hyperopt module
 
 ### How many epochs do I need to get a good Hyperopt result?
@@ -156,7 +172,7 @@ freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossD
 
 ### Why does it take a long time to run hyperopt?
 
-* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/MA9v74M). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you.
+* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/p7nuUNVfP7). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you.
 
 * If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers:
 

docs/hyperopt.md

@@ -237,9 +237,9 @@ class MyAwesomeStrategy(IStrategy):
         dataframe['macdhist'] = macd['macdhist']
 
         bollinger = ta.BBANDS(dataframe, timeperiod=20, nbdevup=2.0, nbdevdn=2.0)
-        dataframe['bb_lowerband'] = boll['lowerband']
-        dataframe['bb_middleband'] = boll['middleband']
-        dataframe['bb_upperband'] = boll['upperband']
+        dataframe['bb_lowerband'] = bollinger['lowerband']
+        dataframe['bb_middleband'] = bollinger['middleband']
+        dataframe['bb_upperband'] = bollinger['upperband']
         return dataframe
 ```
 
@@ -249,15 +249,16 @@ We continue to define hyperoptable parameters:
 
 ```python
 class MyAwesomeStrategy(IStrategy):
-    buy_adx = IntParameter(20, 40, default=30, space="buy")
+    buy_adx = DecimalParameter(20, 40, decimals=1, default=30.1, space="buy")
     buy_rsi = IntParameter(20, 40, default=30, space="buy")
-    buy_adx_enabled = CategoricalParameter([True, False], space="buy")
-    buy_rsi_enabled = CategoricalParameter([True, False], space="buy")
-    buy_trigger = CategoricalParameter(['bb_lower', 'macd_cross_signal'], space="buy")
+    buy_adx_enabled = CategoricalParameter([True, False], default=True, space="buy")
+    buy_rsi_enabled = CategoricalParameter([True, False], default=False, space="buy")
+    buy_trigger = CategoricalParameter(["bb_lower", "macd_cross_signal"], default="bb_lower", space="buy")
 ```
 
-Above definition says: I have five parameters I want to randomly combine to find the best combination.
-Two of them are integer values (`buy_adx` and `buy_rsi`) and I want you test in the range of values 20 to 40.  
+The above definition says: I have five parameters I want to randomly combine to find the best combination.  
+`buy_rsi` is an integer parameter, which will be tested between 20 and 40. This space has a size of 20.  
+`buy_adx` is a decimal parameter, which will be evaluated between 20 and 40 with 1 decimal place (so values are 20.1, 20.2, ...). This space has a size of 200.  
 Then we have three category variables. First two are either `True` or `False`.
 We use these to either enable or disable the ADX and RSI guards.
 The last one we call `trigger` and use it to decide which buy trigger we want to use.
@@ -490,7 +491,7 @@ Given the following result from hyperopt:
 ```
 Best result:
 
-    44/100:    135 trades. Avg profit  0.57%. Total profit  0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367
+    44/100:    135 trades. Avg profit  0.57%. Total profit  0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367
 
     # Buy hyperspace params:
     buy_params = {
@@ -531,7 +532,7 @@ If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'de
 ```
 Best result:
 
-    44/100:    135 trades. Avg profit  0.57%. Total profit  0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367
+    44/100:    135 trades. Avg profit  0.57%. Total profit  0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367
 
     # ROI table:
     minimal_roi = {
@@ -586,7 +587,7 @@ If you are optimizing stoploss values (i.e. if optimization search-space contain
 ```
 Best result:
 
-    44/100:    135 trades. Avg profit  0.57%. Total profit  0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367
+    44/100:    135 trades. Avg profit  0.57%. Total profit  0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367
 
     # Buy hyperspace params:
     buy_params = {
@@ -628,7 +629,7 @@ If you are optimizing trailing stop values (i.e. if optimization search-space co
 ```
 Best result:
 
-    45/100:    606 trades. Avg profit  1.04%. Total profit  0.31555614 BTC ( 630.48Σ%). Avg duration 150.3 mins. Objective: -1.10161
+    45/100:    606 trades. Avg profit  1.04%. Total profit  0.31555614 BTC ( 630.48%). Avg duration 150.3 mins. Objective: -1.10161
 
     # Trailing stop:
     trailing_stop = True

docs/includes/pairlists.md

@@ -112,6 +112,7 @@ The `PriceFilter` allows filtering of pairs by price. Currently the following pr
 
 * `min_price`
 * `max_price`
+* `max_value`
 * `low_price_ratio`
 
 The `min_price` setting removes pairs where the price is below the specified price. This is useful if you wish to avoid trading very low-priced pairs.
@@ -120,6 +121,11 @@ This option is disabled by default, and will only apply if set to > 0.
 The `max_price` setting removes pairs where the price is above the specified price. This is useful if you wish to trade only low-priced pairs.
 This option is disabled by default, and will only apply if set to > 0.
 
+The `max_value` setting removes pairs where the minimum value change is above a specified value.
+This is useful when an exchange has unbalanced limits. For example, if step-size = 1 (so you can only buy 1, or 2, or 3, but not 1.1 Coins) - and the price is pretty high (like 20\$) as the coin has risen sharply since the last limit adaption.
+As a result of the above, you can only buy for 20\$, or 40\$ - but not for 25\$.
+On exchanges that deduct fees from the receiving currency (e.g. FTX) - this can result in high value coins / amounts that are unsellable as the amount is slightly below the limit.
+
 The `low_price_ratio` setting removes pairs where a raise of 1 price unit (pip) is above the `low_price_ratio` ratio.
 This option is disabled by default, and will only apply if set to > 0.
 
@@ -193,7 +199,7 @@ If the volatility over the last 10 days is not in the range of 0.05-0.50, remove
 
 ### Full example of Pairlist Handlers
 
-The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 price unit is > 1%. Then the [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) is applied and pairs are finally shuffled with the random seed set to some predefined value.
+The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#pricefilter), filtering all assets where 1 price unit is > 1%. Then the [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) is applied and pairs are finally shuffled with the random seed set to some predefined value.
 
 ```json
 "exchange": {
@@ -204,7 +210,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets,
     {
         "method": "VolumePairList",
         "number_assets": 20,
-        "sort_key": "quoteVolume",
+        "sort_key": "quoteVolume"
     },
     {"method": "AgeFilter", "min_days_listed": 10},
     {"method": "PrecisionFilter"},

docs/includes/protections.md

@@ -8,7 +8,6 @@ All protection end times are rounded up to the next candle to avoid sudden, unex
 
 !!! Note
     Not all Protections will work for all strategies, and parameters will need to be tuned for your strategy to improve performance.  
-    To align your protection with your strategy, you can define protections in the strategy.
 
 !!! Tip
     Each Protection can be configured multiple times with different parameters, to allow different levels of protection (short-term / long-term).
@@ -47,16 +46,16 @@ This applies across all pairs, unless `only_per_pair` is set to true, which will
 
 The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles.
 
-```json
-"protections": [
+``` python
+protections = [
     {
         "method": "StoplossGuard",
         "lookback_period_candles": 24,
         "trade_limit": 4,
         "stop_duration_candles": 4,
-        "only_per_pair": false
+        "only_per_pair": False
     }
-],
+]
 ```
 
 !!! Note
@@ -69,8 +68,8 @@ The below example stops trading for all pairs for 4 candles after the last trade
 
 The below sample stops trading for 12 candles if max-drawdown is > 20% considering all pairs - with a minimum of `trade_limit` trades - within the last 48 candles. If desired, `lookback_period` and/or `stop_duration` can be used.
 
-```json
-"protections": [
+``` python
+protections = [
       {
         "method": "MaxDrawdown",
         "lookback_period_candles": 48,
@@ -78,7 +77,7 @@ The below sample stops trading for 12 candles if max-drawdown is > 20% consideri
         "stop_duration_candles": 12,
         "max_allowed_drawdown": 0.2
       },
-],
+]
 ```
 
 #### Low Profit Pairs
@@ -88,8 +87,8 @@ If that ratio is below `required_profit`, that pair will be locked for `stop_dur
 
 The below example will stop trading a pair for 60 minutes if the pair does not have a required profit of 2% (and a minimum of 2 trades) within the last 6 candles.
 
-```json
-"protections": [
+``` python
+protections = [
     {
         "method": "LowProfitPairs",
         "lookback_period_candles": 6,
@@ -97,7 +96,7 @@ The below example will stop trading a pair for 60 minutes if the pair does not h
         "stop_duration": 60,
         "required_profit": 0.02
     }
-],
+]
 ```
 
 #### Cooldown Period
@@ -106,13 +105,13 @@ The below example will stop trading a pair for 60 minutes if the pair does not h
 
 The below example will stop trading a pair for 2 candles after closing a trade, allowing this pair to "cool down".
 
-```json
-"protections": [
+``` python
+protections = [
     {
         "method": "CooldownPeriod",
         "stop_duration_candles": 2
     }
-],
+]
 ```
 
 !!! Note
@@ -132,46 +131,6 @@ The below example assumes a timeframe of 1 hour:
 * Locks all pairs that had 4 Trades within the last 6 hours (`6 * 1h candles`) with a combined profit ratio of below 0.02 (<2%) (`LowProfitPairs`).
 * Locks all pairs for 2 candles that had a profit of below 0.01 (<1%) within the last 24h (`24 * 1h candles`), a minimum of 4 trades.
 
-```json
-"timeframe": "1h",
-"protections": [
-    {
-        "method": "CooldownPeriod",
-        "stop_duration_candles": 5
-    },
-    {
-        "method": "MaxDrawdown",
-        "lookback_period_candles": 48,
-        "trade_limit": 20,
-        "stop_duration_candles": 4,
-        "max_allowed_drawdown": 0.2
-    },
-    {
-        "method": "StoplossGuard",
-        "lookback_period_candles": 24,
-        "trade_limit": 4,
-        "stop_duration_candles": 2,
-        "only_per_pair": false
-    },
-    {
-        "method": "LowProfitPairs",
-        "lookback_period_candles": 6,
-        "trade_limit": 2,
-        "stop_duration_candles": 60,
-        "required_profit": 0.02
-    },
-    {
-        "method": "LowProfitPairs",
-        "lookback_period_candles": 24,
-        "trade_limit": 4,
-        "stop_duration_candles": 2,
-        "required_profit": 0.01
-    }
-    ],
-```
-
-You can use the same in your strategy, the syntax is only slightly different:
-
 ``` python
 from freqtrade.strategy import IStrategy
 

docs/index.md

@@ -76,7 +76,7 @@ Alternatively
 
 For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join our slack channel.
 
-Please check out our [discord server](https://discord.gg/MA9v74M).
+Please check out our [discord server](https://discord.gg/p7nuUNVfP7).
 
 You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw).
 

docs/installation.md

@@ -60,7 +60,7 @@ OS Specific steps are listed first, the [Common](#common) section below is neces
     sudo apt-get update
 
     # install packages
-    sudo apt install -y python3-pip python3-venv python3-pandas python3-pip git
+    sudo apt install -y python3-pip python3-venv python3-dev python3-pandas git
     ```
 
 === "RaspberryPi/Raspbian"
@@ -269,7 +269,7 @@ git clone https://github.com/freqtrade/freqtrade.git
 cd freqtrade      
 ```
 
-#### Freqtrade instal: Conda Environment
+#### Freqtrade install: Conda Environment
 
 Prepare conda-freqtrade environment, using file `environment.yml`, which exist in main freqtrade directory
 

docs/overrides/main.html

@@ -0,0 +1,68 @@
+{% extends "base.html" %}
+
+
+<!-- Navigation -->
+{% block site_nav %}
+
+  <!-- Main navigation -->
+  {% if nav %}
+  {% if page and page.meta and page.meta.hide %}
+  {% set hidden = "hidden" if "navigation" in page.meta.hide %}
+  {% endif %}
+  <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" {{ hidden }}>
+    <div class="md-sidebar__scrollwrap">
+      <div id="widget-wrapper">
+
+      </div>
+      <div class="md-sidebar__inner">
+        {% include "partials/nav.html" %}
+      </div>
+    </div>
+  </div>
+  {% endif %}
+
+  <!-- Table of contents -->
+  {% if page.toc and not "toc.integrate" in features %}
+  {% if page and page.meta and page.meta.hide %}
+  {% set hidden = "hidden" if "toc" in page.meta.hide %}
+  {% endif %}
+  <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" {{ hidden }}>
+    <div class="md-sidebar__scrollwrap">
+      <div class="md-sidebar__inner">
+        {% include "partials/toc.html" %}
+      </div>
+    </div>
+  </div>
+  {% endif %}
+{% endblock %}
+
+{% block footer %}
+    {{ super() }}
+
+    <!-- Place this tag in your head or just before your close body tag. -->
+    <script async defer src="https://buttons.github.io/buttons.js"></script>
+    <script src="https://code.jquery.com/jquery-3.4.1.min.js"
+      integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
+
+    <!-- Load binance SDK -->
+    <script async defer src="https://public.bnbstatic.com/static/js/broker-sdk/broker-sdk@1.0.0.min.js"></script>
+
+    <script>
+      window.onload = function () {
+        var sidebar = document.getElementById('widget-wrapper')
+        var newDiv = document.createElement("div");
+        newDiv.id = "widget";
+        try {
+          sidebar.prepend(newDiv);
+
+          window.binanceBrokerPortalSdk.initBrokerSDK('#widget', {
+            apiHost: 'https://www.binance.com',
+            brokerId: 'R4BD3S82',
+            slideTime: 4e4,
+          });
+        } catch(err) {
+          console.log(err)
+        }
+      }
+    </script>
+{% endblock %}

docs/partials/header.html

@@ -1,72 +0,0 @@
-{#-
-This file was automatically generated - do not edit
--#}
-{% set site_url = config.site_url | d(nav.homepage.url, true) | url %}
-{% if not config.use_directory_urls and site_url[0] == site_url[-1] == "." %}
-{% set site_url = site_url ~ "/index.html" %}
-{% endif %}
-<header class="md-header" data-md-component="header">
-  <nav class="md-header__inner md-grid" aria-label="{{ lang.t('header.title') }}">
-    <a href="{{ site_url }}" title="{{ config.site_name | e }}" class="md-header__button md-logo"
-      aria-label="{{ config.site_name }}">
-      {% include "partials/logo.html" %}
-    </a>
-    <label class="md-header__button md-icon" for="__drawer">
-      {% include ".icons/material/menu" ~ ".svg" %}
-    </label>
-    <div class="md-header__title" data-md-component="header-title">
-      <div class="md-header__ellipsis">
-        <div class="md-header__topic">
-          <span class="md-ellipsis">
-            {{ config.site_name }}
-          </span>
-        </div>
-        <div class="md-header__topic" data-md-component="header-topic">
-          <span class="md-ellipsis">
-            {% if page and page.meta and page.meta.title %}
-            {{ page.meta.title }}
-            {% else %}
-            {{ page.title }}
-            {% endif %}
-          </span>
-        </div>
-      </div>
-    </div>
-    <div class="md-header__options">
-      {% if config.extra.alternate %}
-      <div class="md-select">
-        {% set icon = config.theme.icon.alternate or "material/translate" %}
-        <span class="md-header__button md-icon">
-          {% include ".icons/" ~ icon ~ ".svg" %}
-        </span>
-        <div class="md-select__inner">
-          <ul class="md-select__list">
-            {% for alt in config.extra.alternate %}
-            <li class="md-select__item">
-              <a href="{{ alt.link | url }}" class="md-select__link">
-                {{ alt.name }}
-              </a>
-            </li>
-            {% endfor %}
-          </ul>
-        </div>
-      </div>
-      {% endif %}
-    </div>
-    {% if "search" in config["plugins"] %}
-    <label class="md-header__button md-icon" for="__search">
-      {% include ".icons/material/magnify.svg" %}
-    </label>
-    {% include "partials/search.html" %}
-    {% endif %}
-    {% if config.repo_url %}
-    <div class="md-header__source">
-      {% include "partials/source.html" %}
-    </div>
-    {% endif %}
-  </nav>
-    <!-- Place this tag in your head or just before your close body tag. -->
-    <script async defer src="https://buttons.github.io/buttons.js"></script>
-    <script src="https://code.jquery.com/jquery-3.4.1.min.js"
-      integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-</header>

docs/plotting.md

@@ -170,9 +170,15 @@ Additional features when using plot_config include:
 * Specify additional subplots
 * Specify indicator pairs to fill area in between 
 
-The sample plot configuration below specifies fixed colors for the indicators. Otherwise consecutive plots may produce different colorschemes each time, making comparisons difficult.
+The sample plot configuration below specifies fixed colors for the indicators. Otherwise, consecutive plots may produce different color schemes each time, making comparisons difficult.
 It also allows multiple subplots to display both MACD and RSI at the same time.
 
+Plot type can be configured using `type` key. Possible types are:
+* `scatter` corresponding to `plotly.graph_objects.Scatter` class (default).
+* `bar` corresponding to `plotly.graph_objects.Bar` class.
+
+Extra parameters to `plotly.graph_objects.*` constructor can be specified in `plotly` dict.
+
 Sample configuration with inline comments explaining the process:
 
 ``` python
@@ -198,7 +204,8 @@ Sample configuration with inline comments explaining the process:
             # Create subplot MACD
             "MACD": {
                 'macd': {'color': 'blue', 'fill_to': 'macdhist'},
-                'macdsignal': {'color': 'orange'}
+                'macdsignal': {'color': 'orange'},
+                'macdhist': {'type': 'bar', 'plotly': {'opacity': 0.9}}
             },
             # Additional subplot RSI
             "RSI": {
@@ -213,6 +220,9 @@ Sample configuration with inline comments explaining the process:
     The above configuration assumes that `ema10`, `ema50`, `senkou_a`, `senkou_b`,
     `macd`, `macdsignal`, `macdhist` and `rsi` are columns in the DataFrame created by the strategy.
 
+!!! Warning
+    `plotly` arguments are only supported with plotly library and will not work with freq-ui.
+
 ## Plot profit
 
 ![plot-profit](assets/plot-profit.png)
@@ -265,6 +275,7 @@ optional arguments:
                         (backtest file)) Default: file
   -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME
                         Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).
+  --auto-open           Automatically open generated plot.
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).

docs/requirements-docs.txt

@@ -1,3 +1,4 @@
-mkdocs-material==7.1.3
+mkdocs==1.2.1
+mkdocs-material==7.1.8
 mdx_truly_sane_lists==1.2
-pymdown-extensions==8.1.1
+pymdown-extensions==8.2

docs/rest-api.md

@@ -71,7 +71,10 @@ If you run your bot using docker, you'll need to have the bot listen to incoming
     "api_server": {
         "enabled": true,
         "listen_ip_address": "0.0.0.0",
-        "listen_port": 8080
+        "listen_port": 8080,
+        "username": "Freqtrader",
+        "password": "SuperSecret1!",
+        //...
     },
 ```
 
@@ -106,7 +109,10 @@ By default, the script assumes `127.0.0.1` (localhost) and port `8080` to be use
     "api_server": {
         "enabled": true,
         "listen_ip_address": "0.0.0.0",
-        "listen_port": 8080
+        "listen_port": 8080,
+        "username": "Freqtrader",
+        "password": "SuperSecret1!",
+        //...
     }
 }
 ```

docs/sql_cheatsheet.md

@@ -19,7 +19,7 @@ The freqtrade docker image does contain sqlite3, so you can edit the database wi
 
 ``` bash
 docker-compose exec freqtrade /bin/bash
-sqlite3 <databasefile>.sqlite
+sqlite3 <database-file>.sqlite
 ```
 
 ## Open the DB
@@ -99,3 +99,32 @@ DELETE FROM trades WHERE id = 31;
 
 !!! Warning
     This will remove this trade from the database. Please make sure you got the correct id and **NEVER** run this query without the `where` clause.
+
+## Use a different database system
+
+!!! Warning
+    By using one of the below database systems, you acknowledge that you know how to manage such a system. Freqtrade will not provide any support with setup or maintenance (or backups) of the below database systems.
+
+### PostgreSQL
+
+Freqtrade supports PostgreSQL by using SQLAlchemy, which supports multiple different database systems.
+
+Installation:
+`pip install psycopg2`
+
+Usage:
+`... --db-url postgresql+psycopg2://<username>:<password>@localhost:5432/<database>`
+
+Freqtrade will automatically create the tables necessary upon startup.
+
+If you're running different instances of Freqtrade, you must either setup one database per Instance or use different users / schemas for your connections.
+
+### MariaDB / MySQL
+
+Freqtrade supports MariaDB by using SQLAlchemy, which supports multiple different database systems.
+
+Installation:
+`pip install pymysql`
+
+Usage:
+`... --db-url mysql+pymysql://<username>:<password>@localhost:3306/<database>`

docs/strategy-advanced.md

@@ -40,34 +40,79 @@ class AwesomeStrategy(IStrategy):
 !!! Note
     If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
 
-***
+## Dataframe access
 
-### Storing custom information using DatetimeIndex from `dataframe`
+You may access dataframe in various strategy functions by querying it from dataprovider.
 
-Imagine you need to store an indicator like `ATR` or `RSI` into `custom_info`. To use this in a meaningful way, you will not only need the raw data of the indicator, but probably also need to keep the right timestamps.
+``` python
+from freqtrade.exchange import timeframe_to_prev_date
 
-```python
-import talib.abstract as ta
 class AwesomeStrategy(IStrategy):
-    # Create custom dictionary
-    custom_info = {}
+    def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: float,
+                           rate: float, time_in_force: str, sell_reason: str,
+                           current_time: 'datetime', **kwargs) -> bool:
+        # Obtain pair dataframe.
+        dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
 
-    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
-        # using "ATR" here as example
-        dataframe['atr'] = ta.ATR(dataframe)
-        if self.dp.runmode.value in ('backtest', 'hyperopt'):
-          # add indicator mapped to correct DatetimeIndex to custom_info
-          self.custom_info[metadata['pair']] = dataframe[['date', 'atr']].set_index('date')
-        return dataframe
+        # Obtain last available candle. Do not use current_time to look up latest candle, because 
+        # current_time points to curret incomplete candle whose data is not available.
+        last_candle = dataframe.iloc[-1].squeeze()
+        # <...>
+
+        # In dry/live runs trade open date will not match candle open date therefore it must be 
+        # rounded.
+        trade_date = timeframe_to_prev_date(self.timeframe, trade.open_date_utc)
+        # Look up trade candle.
+        trade_candle = dataframe.loc[dataframe['date'] == trade_date]
+        # trade_candle may be empty for trades that just opened as it is still incomplete.
+        if not trade_candle.empty:
+            trade_candle = trade_candle.squeeze()
+            # <...>
 ```
 
-!!! Warning
-    The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
+!!! Warning "Using .iloc[-1]"
+    You can use `.iloc[-1]` here because `get_analyzed_dataframe()` only returns candles that backtesting is allowed to see.
+    This will not work in `populate_*` methods, so make sure to not use `.iloc[]` in that area.
+    Also, this will only work starting with version 2021.5.
+
+***
+
+## Custom sell signal
+
+It is possible to define custom sell signals, indicating that specified position should be sold. This is very useful when we need to customize sell conditions for each individual trade, or if you need the trade profit to take the sell decision. 
+
+For example you could implement a 1:2 risk-reward ROI with `custom_sell()`.
+
+Using custom_sell() signals in place of stoplosses though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange.
 
 !!! Note
-    If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
+    Returning a `string` or `True` from this method is equal to setting sell signal on a candle at specified time. This method is not called when sell signal is set already, or if sell signals are disabled (`use_sell_signal=False` or `sell_profit_only=True` while profit is below `sell_profit_offset`). `string` max length is 64 characters. Exceeding this limit will cause the message to be truncated to 64 characters.
 
-See `custom_stoploss` examples below on how to access the saved dataframe columns
+An example of how we can use different indicators depending on the current profit and also sell trades that were open longer than one day:
+
+``` python
+class AwesomeStrategy(IStrategy):
+    def custom_sell(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float,
+                    current_profit: float, **kwargs):
+        dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
+        last_candle = dataframe.iloc[-1].squeeze()
+
+        # Above 20% profit, sell when rsi < 80
+        if current_profit > 0.2:
+            if last_candle['rsi'] < 80:
+                return 'rsi_below_80'
+
+        # Between 2% and 10%, sell if EMA-long above EMA-short
+        if 0.02 < current_profit < 0.1:
+            if last_candle['emalong'] > last_candle['emashort']:
+                return 'ema_long_below_80'
+
+        # Sell any positions at a loss if they are held for more than one day.
+        if current_profit < 0.0 and (current_time - trade.open_date_utc).days >= 1:
+            return 'unclog'
+```
+
+See [Dataframe access](#dataframe-access) for more information about dataframe use in strategy callbacks.
 
 ## Custom stoploss
 
@@ -110,7 +155,7 @@ class AwesomeStrategy(IStrategy):
         :param current_rate: Rate, calculated based on pricing settings in ask_strategy.
         :param current_profit: Current profit (as ratio), calculated based on current_rate.
         :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
-        :return float: New stoploss value, relative to the currentrate
+        :return float: New stoploss value, relative to the current rate
         """
         return -0.04
 ```
@@ -222,7 +267,6 @@ Instead of continuously trailing behind the current price, this example sets fix
 * Once profit is > 25% - set stoploss to 15% above open price.
 * Once profit is > 40% - set stoploss to 25% above open price.
 
-
 ``` python
 from datetime import datetime
 from freqtrade.persistence import Trade
@@ -248,63 +292,46 @@ class AwesomeStrategy(IStrategy):
         # return maximum stoploss value, keeping current stoploss price unchanged
         return 1
 ```
+
 #### Custom stoploss using an indicator from dataframe example
 
-Imagine you want to use `custom_stoploss()` to use a trailing indicator like e.g. "ATR"
-
-See: "Storing custom information using DatetimeIndex from `dataframe`" example above) on how to store the indicator into `custom_info`
-
-!!! Warning
-    only use .iat[-1] in live mode, not in backtesting/hyperopt
-    otherwise you will look into the future
-    see [Common mistakes when developing strategies](strategy-customization.md#common-mistakes-when-developing-strategies) for more info.
+Absolute stoploss value may be derived from indicators stored in dataframe. Example uses parabolic SAR below the price as stoploss.
 
 ``` python
-from freqtrade.persistence import Trade
-from freqtrade.state import RunMode
-
 class AwesomeStrategy(IStrategy):
 
-    # ... populate_* methods
+    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        # <...>
+        dataframe['sar'] = ta.SAR(dataframe)
 
     use_custom_stoploss = True
 
     def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
                         current_rate: float, current_profit: float, **kwargs) -> float:
 
-        result = 1
-        if self.custom_info and pair in self.custom_info and trade:
-            # using current_time directly (like below) will only work in backtesting.
-            # so check "runmode" to make sure that it's only used in backtesting/hyperopt
-            if self.dp and self.dp.runmode.value in ('backtest', 'hyperopt'):
-              relative_sl = self.custom_info[pair].loc[current_time]['atr']
-            # in live / dry-run, it'll be really the current time
-            else:
-              # but we can just use the last entry from an already analyzed dataframe instead
-              dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair,
-                                                                       timeframe=self.timeframe)
-              # WARNING
-              # only use .iat[-1] in live mode, not in backtesting/hyperopt
-              # otherwise you will look into the future
-              # see: https://www.freqtrade.io/en/latest/strategy-customization/#common-mistakes-when-developing-strategies
-              relative_sl = dataframe['atr'].iat[-1]
+        dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
+        last_candle = dataframe.iloc[-1].squeeze()
 
-            if (relative_sl is not None):
-                # new stoploss relative to current_rate
-                new_stoploss = (current_rate-relative_sl)/current_rate
-                # turn into relative negative offset required by `custom_stoploss` return implementation
-                result = new_stoploss - 1
+        # Use parabolic sar as absolute stoploss price
+        stoploss_price = last_candle['sar']
 
-        return result
+        # Convert absolute price to percentage relative to current_rate
+        if stoploss_price < current_rate:
+            return (stoploss_price / current_rate) - 1
+
+        # return maximum stoploss value, keeping current stoploss price unchanged
+        return 1
 ```
 
+See [Dataframe access](#dataframe-access) for more information about dataframe use in strategy callbacks.
+
 ---
 
 ## Custom order timeout rules
 
 Simple, time-based order-timeouts can be configured either via strategy or in the configuration in the `unfilledtimeout` section.
 
-However, freqtrade also offers a custom callback for both order types, which allows you to decide based on custom criteria if a order did time out or not.
+However, freqtrade also offers a custom callback for both order types, which allows you to decide based on custom criteria if an order did time out or not.
 
 !!! Note
     Unfilled order timeouts are not relevant during backtesting or hyperopt, and are only relevant during real (live) trading. Therefore these methods are only called in these circumstances.
@@ -530,7 +557,7 @@ Both attributes and methods may be overridden, altering behavior of the original
 
 ## Embedding Strategies
 
-Freqtrade provides you with with an easy way to embed the strategy into your configuration file.
+Freqtrade provides you with an easy way to embed the strategy into your configuration file.
 This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field,
 in your chosen config file.
 

docs/strategy-customization.md

@@ -159,7 +159,7 @@ Edit the method `populate_buy_trend()` in your strategy file to update your buy
 
 It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected.
 
-This will method will also define a new column, `"buy"`, which needs to contain 1 for buys, and 0 for "no action".
+This method will also define a new column, `"buy"`, which needs to contain 1 for buys, and 0 for "no action".
 
 Sample from `user_data/strategies/sample_strategy.py`:
 
@@ -193,7 +193,7 @@ Please note that the sell-signal is only used if `use_sell_signal` is set to tru
 
 It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected.
 
-This will method will also define a new column, `"sell"`, which needs to contain 1 for sells, and 0 for "no action".
+This method will also define a new column, `"sell"`, which needs to contain 1 for sells, and 0 for "no action".
 
 Sample from `user_data/strategies/sample_strategy.py`:
 
@@ -422,10 +422,6 @@ if self.dp:
     Returns an empty dataframe if the requested pair was not cached.
     This should not happen when using whitelisted pairs.
 
-
-!!! Warning "Warning about backtesting"
-    This method will return an empty dataframe during backtesting.
-
 ### *orderbook(pair, maximum)*
 
 ``` python
@@ -633,7 +629,7 @@ Stoploss values returned from `custom_stoploss` must specify a percentage relati
         def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
                             current_rate: float, current_profit: float, **kwargs) -> float:
 
-            # once the profit has risin above 10%, keep the stoploss at 7% above the open price
+            # once the profit has risen above 10%, keep the stoploss at 7% above the open price
             if current_profit > 0.10:
                 return stoploss_from_open(0.07, current_profit)
 

docs/stylesheets/ft.extra.css

@@ -11,3 +11,18 @@
 .rst-versions .rst-other-versions {
     color: white;
 }
+
+
+#widget-wrapper {
+    height: calc(220px * 0.5625 + 18px);
+    width: 220px;
+    margin: 0 auto 16px auto;
+    border-style: solid;
+    border-color: var(--md-code-bg-color);
+    border-width: 1px;
+    border-radius: 5px;
+}
+
+@media screen and (max-width: calc(76.25em - 1px)) {
+    #widget-wrapper { display: none; }
+}

docs/telegram-usage.md

@@ -72,22 +72,32 @@ Example configuration showing the different settings:
 
 ``` json
 "telegram": {
-      "enabled": true,
-      "token": "your_telegram_token",
-      "chat_id": "your_telegram_chat_id",
-      "notification_settings": {
-         "status": "silent",
-         "warning": "on",
-         "startup": "off",
-         "buy": "silent",
-         "sell": "on",
-         "buy_cancel": "silent",
-         "sell_cancel": "on",
-         "buy_fill": "off",
-         "sell_fill": "off"
-      },
-      "balance_dust_level": 0.01
-   },
+    "enabled": true,
+    "token": "your_telegram_token",
+    "chat_id": "your_telegram_chat_id",
+    "notification_settings": {
+        "status": "silent",
+        "warning": "on",
+        "startup": "off",
+        "buy": "silent",
+        "sell": {
+            "roi": "silent",
+            "emergency_sell": "on",
+            "force_sell": "on",
+            "sell_signal": "silent",
+            "trailing_stop_loss": "on",
+            "stop_loss": "on",
+            "stoploss_on_exchange": "on",
+            "custom_sell": "silent"
+        },
+        "buy_cancel": "silent",
+        "sell_cancel": "on",
+        "buy_fill": "off",
+        "sell_fill": "off"
+    },
+    "reload": true,
+    "balance_dust_level": 0.01
+},
 ```
 
 `buy` notifications are sent when the order is placed, while `buy_fill` notifications are sent when the order is filled on the exchange.
@@ -96,6 +106,7 @@ Example configuration showing the different settings:
 
 
 `balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown.
+`reload` allows you to disable reload-buttons on selected messages.
 
 ## Create a custom keyboard (command shortcut buttons)
 
@@ -154,7 +165,7 @@ official commands. You can ask at any moment for help with `/help`.
 | `/count` | Displays number of trades used and available
 | `/locks` | Show currently locked pairs.
 | `/unlock <pair or lock_id>` | Remove the lock for this pair (or for this lock id).
-| `/profit` | Display a summary of your profit/loss from close trades and some stats about your performance
+| `/profit [<n>]` | Display a summary of your profit/loss from close trades and some stats about your performance, over the last n days (all trades by default)
 | `/forcesell <trade_id>` | Instantly sells the given trade  (Ignoring `minimum_roi`).
 | `/forcesell all` | Instantly sells all open trades (Ignoring `minimum_roi`).
 | `/forcebuy <pair> [rate]` | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True)
@@ -250,10 +261,14 @@ Return a summary of your profit/loss and performance.
 
 > **BITTREX:** Selling BTC/LTC with limit `0.01650000 (profit: ~-4.07%, -0.00008168)`
 
-### /forcebuy <pair>
+### /forcebuy <pair> [rate]
 
 > **BITTREX:** Buying ETH/BTC with limit `0.03400000` (`1.000000 ETH`, `225.290 USD`)
 
+Omitting the pair will open a query asking for the pair to buy (based on the current whitelist).
+
+![Telegram force-buy screenshot](assets/telegram_forcebuy.png)
+
 Note that for this to work, `forcebuy_enable` needs to be set to true.
 
 [More details](configuration.md#understand-forcebuy_enable)
@@ -261,12 +276,12 @@ Note that for this to work, `forcebuy_enable` needs to be set to true.
 ### /performance
 
 Return the performance of each crypto-currency the bot has sold.
-> Performance:
-> 1. `RCN/BTC 57.77%`  
-> 2. `PAY/BTC 56.91%`  
-> 3. `VIB/BTC 47.07%`  
-> 4. `SALT/BTC 30.24%`  
-> 5. `STORJ/BTC 27.24%`  
+> Performance:  
+> 1. `RCN/BTC 0.003 BTC (57.77%) (1)`  
+> 2. `PAY/BTC 0.0012 BTC (56.91%) (1)`  
+> 3. `VIB/BTC 0.0011 BTC (47.07%) (1)`  
+> 4. `SALT/BTC 0.0010 BTC (30.24%) (1)`  
+> 5. `STORJ/BTC 0.0009 BTC (27.24%) (1)`  
 > ...  
 
 ### /balance

docs/windows_installation.md

@@ -1,3 +1,5 @@
+# Windows installation
+
 We **strongly** recommend that Windows users use [Docker](docker_quickstart.md) as this will work much easier and smoother (also more secure).
 
 If that is not possible, try using the Windows Linux subsystem (WSL) - for which the Ubuntu instructions should work.
@@ -21,7 +23,7 @@ git clone https://github.com/freqtrade/freqtrade.git
 
 Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows).
 
-As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.19‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version)
+As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial pre-compiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.20‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version).
 
 Freqtrade provides these dependencies for the latest 2 Python versions (3.7 and 3.8) and for 64bit Windows.
 Other versions must be downloaded from the above link.

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2021.4'
+__version__ = '2021.6'
 
 if __version__ == 'develop':
 

freqtrade/commands/arguments.py

@@ -69,7 +69,7 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
                        "timerange", "timeframe", "no_trades"]
 
 ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
-                    "trade_source", "timeframe"]
+                    "trade_source", "timeframe", "plot_auto_open"]
 
 ARGS_INSTALL_UI = ["erase_ui_only"]
 

freqtrade/commands/build_config_commands.py

@@ -183,7 +183,7 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None:
     """
     Applies selections to the template and writes the result to config_path
     :param config_path: Path object for new config file. Should not exist yet
-    :param selecions: Dict containing selections taken by the user.
+    :param selections: Dict containing selections taken by the user.
     """
     from jinja2.exceptions import TemplateNotFound
     try:
@@ -213,7 +213,7 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None:
 def start_new_config(args: Dict[str, Any]) -> None:
     """
     Create a new strategy from a template
-    Asking the user questions to fill out the templateaccordingly.
+    Asking the user questions to fill out the template accordingly.
     """
 
     config_path = Path(args['config'][0])

freqtrade/commands/cli_options.py

@@ -167,8 +167,9 @@ AVAILABLE_CLI_OPTIONS = {
     ),
     "export": Arg(
         '--export',
-        help='Export backtest results, argument are: trades. '
-        'Example: `--export=trades`',
+        help='Export backtest results (default: trades).',
+        choices=constants.EXPORT_OPTIONS,
+
     ),
     "exportfilename": Arg(
         '--export-filename',
@@ -433,6 +434,11 @@ AVAILABLE_CLI_OPTIONS = {
         metavar='INT',
         default=750,
     ),
+    "plot_auto_open": Arg(
+        '--auto-open',
+        help='Automatically open generated plot.',
+        action='store_true',
+    ),
     "no_trades": Arg(
         '--no-trades',
         help='Skip using trades from backtesting file and DB.',

freqtrade/commands/data_commands.py

@@ -8,11 +8,11 @@ from freqtrade.configuration import TimeRange, setup_utils_configuration
 from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format
 from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data,
                                     refresh_backtest_trades_data)
+from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
-from freqtrade.state import RunMode
 
 
 logger = logging.getLogger(__name__)

freqtrade/commands/deploy_commands.py

@@ -8,9 +8,9 @@ import requests
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.configuration.directory_operations import copy_sample_files, create_userdata_dir
 from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
+from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import render_template, render_template_with_fallback
-from freqtrade.state import RunMode
 
 
 logger = logging.getLogger(__name__)

freqtrade/commands/hyperopt_commands.py

@@ -6,8 +6,9 @@ from colorama import init as colorama_init
 
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.data.btanalysis import get_latest_hyperopt_file
+from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
-from freqtrade.state import RunMode
+from freqtrade.optimize.optimize_reports import show_backtest_result
 
 
 logger = logging.getLogger(__name__)
@@ -66,7 +67,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     if epochs and not no_details:
         sorted_epochs = sorted(epochs, key=itemgetter('loss'))
         results = sorted_epochs[0]
-        HyperoptTools.print_epoch_details(results, total_epochs, print_json, no_header)
+        HyperoptTools.show_epoch_details(results, total_epochs, print_json, no_header)
 
     if epochs and export_csv:
         HyperoptTools.export_csv_file(
@@ -125,18 +126,26 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
 
     if epochs:
         val = epochs[n]
-        HyperoptTools.print_epoch_details(val, total_epochs, print_json, no_header,
-                                          header_str="Epoch details")
+
+        metrics = val['results_metrics']
+        if 'strategy_name' in metrics:
+            show_backtest_result(metrics['strategy_name'], metrics,
+                                 metrics['stake_currency'])
+
+        HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header,
+                                         header_str="Epoch details")
 
 
 def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
     """
     Filter our items from the list of hyperopt results
+    TODO: after 2021.5 remove all "legacy" mode queries.
     """
     if filteroptions['only_best']:
         epochs = [x for x in epochs if x['is_best']]
     if filteroptions['only_profitable']:
-        epochs = [x for x in epochs if x['results_metrics']['profit'] > 0]
+        epochs = [x for x in epochs if x['results_metrics'].get(
+            'profit', x['results_metrics'].get('profit_total', 0)) > 0]
 
     epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
 
@@ -153,34 +162,59 @@ def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
     return epochs
 
 
+def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int):
+    """
+    Filter epochs with trade-counts > trades
+    """
+    return [
+        x for x in epochs
+        if x['results_metrics'].get(
+            'trade_count', x['results_metrics'].get('total_trades', 0)
+        ) > trade_count
+    ]
+
+
 def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
 
     if filteroptions['filter_min_trades'] > 0:
-        epochs = [
-            x for x in epochs
-            if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades']
-        ]
+        epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades'])
+
     if filteroptions['filter_max_trades'] > 0:
         epochs = [
             x for x in epochs
-            if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades']
+            if x['results_metrics'].get(
+                'trade_count', x['results_metrics'].get('total_trades')
+                ) < filteroptions['filter_max_trades']
         ]
     return epochs
 
 
 def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
 
+    def get_duration_value(x):
+        # Duration in minutes ...
+        if 'duration' in x['results_metrics']:
+            return x['results_metrics']['duration']
+        else:
+            # New mode
+            if 'holding_avg_s' in x['results_metrics']:
+                avg = x['results_metrics']['holding_avg_s']
+                return avg // 60
+            raise OperationalException(
+                "Holding-average not available. Please omit the filter on average time, "
+                "or rerun hyperopt with this version")
+
     if filteroptions['filter_min_avg_time'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
         epochs = [
             x for x in epochs
-            if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time']
+            if get_duration_value(x) > filteroptions['filter_min_avg_time']
         ]
     if filteroptions['filter_max_avg_time'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
         epochs = [
             x for x in epochs
-            if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time']
+            if get_duration_value(x) < filteroptions['filter_max_avg_time']
         ]
 
     return epochs
@@ -189,28 +223,36 @@ def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
 def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
 
     if filteroptions['filter_min_avg_profit'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
         epochs = [
             x for x in epochs
-            if x['results_metrics']['avg_profit'] > filteroptions['filter_min_avg_profit']
+            if x['results_metrics'].get(
+                'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
+            ) > filteroptions['filter_min_avg_profit']
         ]
     if filteroptions['filter_max_avg_profit'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
         epochs = [
             x for x in epochs
-            if x['results_metrics']['avg_profit'] < filteroptions['filter_max_avg_profit']
+            if x['results_metrics'].get(
+                'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
+                ) < filteroptions['filter_max_avg_profit']
         ]
     if filteroptions['filter_min_total_profit'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
         epochs = [
             x for x in epochs
-            if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit']
+            if x['results_metrics'].get(
+                'profit', x['results_metrics'].get('profit_total_abs', 0)
+                ) > filteroptions['filter_min_total_profit']
         ]
     if filteroptions['filter_max_total_profit'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
         epochs = [
             x for x in epochs
-            if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit']
+            if x['results_metrics'].get(
+                'profit', x['results_metrics'].get('profit_total_abs', 0)
+                ) < filteroptions['filter_max_total_profit']
         ]
     return epochs
 
@@ -218,11 +260,11 @@ def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
 def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
 
     if filteroptions['filter_min_objective'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
 
         epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
     if filteroptions['filter_max_objective'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
 
         epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
 

freqtrade/commands/list_commands.py

@@ -1,7 +1,6 @@
 import csv
 import logging
 import sys
-from collections import OrderedDict
 from pathlib import Path
 from typing import Any, Dict, List
 
@@ -12,11 +11,11 @@ from tabulate import tabulate
 
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
+from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import market_is_active, validate_exchanges
 from freqtrade.misc import plural
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
-from freqtrade.state import RunMode
 
 
 logger = logging.getLogger(__name__)
@@ -54,15 +53,21 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
         reset = ''
 
     names = [s['name'] for s in objs]
-    objss_to_print = [{
+    objs_to_print = [{
         'name': s['name'] if s['name'] else "--",
         'location': s['location'].name,
         'status': (red + "LOAD FAILED" + reset if s['class'] is None
                    else "OK" if names.count(s['name']) == 1
                    else yellow + "DUPLICATE NAME" + reset)
     } for s in objs]
-
-    print(tabulate(objss_to_print, headers='keys', tablefmt='psql', stralign='right'))
+    for idx, s in enumerate(objs):
+        if 'hyperoptable' in s:
+            objs_to_print[idx].update({
+                'hyperoptable': "Yes" if s['hyperoptable']['count'] > 0 else "No",
+                'buy-Params': len(s['hyperoptable'].get('buy', [])),
+                'sell-Params': len(s['hyperoptable'].get('sell', [])),
+            })
+    print(tabulate(objs_to_print, headers='keys', tablefmt='psql', stralign='right'))
 
 
 def start_list_strategies(args: Dict[str, Any]) -> None:
@@ -75,6 +80,11 @@ def start_list_strategies(args: Dict[str, Any]) -> None:
     strategy_objs = StrategyResolver.search_all_objects(directory, not args['print_one_column'])
     # Sort alphabetically
     strategy_objs = sorted(strategy_objs, key=lambda x: x['name'])
+    for obj in strategy_objs:
+        if obj['class']:
+            obj['hyperoptable'] = obj['class'].detect_all_parameters()
+        else:
+            obj['hyperoptable'] = {'count': 0}
 
     if args['print_one_column']:
         print('\n'.join([s['name'] for s in strategy_objs]))
@@ -143,7 +153,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
                                      pairs_only=pairs_only,
                                      active_only=active_only)
         # Sort the pairs/markets by symbol
-        pairs = OrderedDict(sorted(pairs.items()))
+        pairs = dict(sorted(pairs.items()))
     except Exception as e:
         raise OperationalException(f"Cannot get markets. Reason: {e}") from e
 

freqtrade/commands/optimize_commands.py

@@ -3,9 +3,9 @@ from typing import Any, Dict
 
 from freqtrade import constants
 from freqtrade.configuration import setup_utils_configuration
+from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import round_coin_value
-from freqtrade.state import RunMode
 
 
 logger = logging.getLogger(__name__)
@@ -15,6 +15,7 @@ def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[
     """
     Prepare the configuration for the Hyperopt module
     :param args: Cli args from Arguments()
+    :param method: Bot running mode
     :return: Configuration
     """
     config = setup_utils_configuration(args, method)

freqtrade/commands/pairlist_commands.py

@@ -4,8 +4,8 @@ from typing import Any, Dict
 import rapidjson
 
 from freqtrade.configuration import setup_utils_configuration
+from freqtrade.enums import RunMode
 from freqtrade.resolvers import ExchangeResolver
-from freqtrade.state import RunMode
 
 
 logger = logging.getLogger(__name__)
@@ -31,7 +31,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None:
         results[curr] = pairlists.whitelist
 
     for curr, pairlist in results.items():
-        if not args.get('print_one_column', False):
+        if not args.get('print_one_column', False) and not args.get('list_pairs_print_json', False):
             print(f"Pairs for {curr}: ")
 
         if args.get('print_one_column', False):

freqtrade/commands/plot_commands.py

@@ -1,8 +1,8 @@
 from typing import Any, Dict
 
 from freqtrade.configuration import setup_utils_configuration
+from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
-from freqtrade.state import RunMode
 
 
 def validate_plot_args(args: Dict[str, Any]) -> None:

freqtrade/configuration/check_exchange.py

@@ -1,10 +1,10 @@
 import logging
 from typing import Any, Dict
 
+from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt,
                                 is_exchange_officially_supported, validate_exchange)
-from freqtrade.state import RunMode
 
 
 logger = logging.getLogger(__name__)

freqtrade/configuration/config_setup.py

@@ -1,7 +1,7 @@
 import logging
 from typing import Any, Dict
 
-from freqtrade.state import RunMode
+from freqtrade.enums import RunMode
 
 from .check_exchange import remove_credentials
 from .config_validation import validate_config_consistency
@@ -15,6 +15,7 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str
     """
     Prepare the configuration for utils subcommands
     :param args: Cli args from Arguments()
+    :param method: Bot running mode
     :return: Configuration
     """
     configuration = Configuration(args, method)

freqtrade/configuration/config_validation.py

@@ -6,8 +6,8 @@ from jsonschema import Draft4Validator, validators
 from jsonschema.exceptions import ValidationError, best_match
 
 from freqtrade import constants
+from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
-from freqtrade.state import RunMode
 
 
 logger = logging.getLogger(__name__)

freqtrade/configuration/configuration.py

@@ -12,10 +12,10 @@ from freqtrade.configuration.check_exchange import check_exchange
 from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
 from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
 from freqtrade.configuration.load_config import load_config_file, load_file
+from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.loggers import setup_logging
 from freqtrade.misc import deep_merge_dicts
-from freqtrade.state import NON_UTIL_MODES, TRADING_MODES, RunMode
 
 
 logger = logging.getLogger(__name__)
@@ -375,6 +375,9 @@ class Configuration:
         self._args_to_config(config, argname='plot_limit',
                              logstring='Limiting plot to: {}')
 
+        self._args_to_config(config, argname='plot_auto_open',
+                             logstring='Parameter --auto-open detected.')
+
         self._args_to_config(config, argname='trade_source',
                              logstring='Using trades from: {}')
 
@@ -457,7 +460,7 @@ class Configuration:
             pairs_file = Path(self.args["pairs_file"])
             logger.info(f'Reading pairs file "{pairs_file}".')
             # Download pairs from the pairs file if no config is specified
-            # or if pairs file is specified explicitely
+            # or if pairs file is specified explicitly
             if not pairs_file.exists():
                 raise OperationalException(f'No pairs file found with path "{pairs_file}".')
             config['pairs'] = load_file(pairs_file)

freqtrade/configuration/load_config.py

@@ -43,7 +43,7 @@ def load_file(path: Path) -> Dict[str, Any]:
         with path.open('r') as file:
             config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE)
     except FileNotFoundError:
-        raise OperationalException(f'File file "{path}" not found!')
+        raise OperationalException(f'File "{path}" not found!')
     return config
 
 

freqtrade/configuration/timerange.py

@@ -3,6 +3,7 @@ This module contains the argument manager class
 """
 import logging
 import re
+from datetime import datetime
 from typing import Optional
 
 import arrow
@@ -43,7 +44,7 @@ class TimeRange:
             self.startts = self.startts - seconds
 
     def adjust_start_if_necessary(self, timeframe_secs: int, startup_candles: int,
-                                  min_date: arrow.Arrow) -> None:
+                                  min_date: datetime) -> None:
         """
         Adjust startts by <startup_candles> candles.
         Applies only if no startup-candles have been available.
@@ -54,11 +55,11 @@ class TimeRange:
         :return: None (Modifies the object in place)
         """
         if (not self.starttype or (startup_candles
-                                   and min_date.int_timestamp >= self.startts)):
+                                   and min_date.timestamp() >= self.startts)):
             # If no startts was defined, or backtest-data starts at the defined backtest-date
             logger.warning("Moving start-date by %s candles to account for startup time.",
                            startup_candles)
-            self.startts = (min_date.int_timestamp + timeframe_secs * startup_candles)
+            self.startts = int(min_date.timestamp() + timeframe_secs * startup_candles)
             self.starttype = 'date'
 
     @staticmethod

freqtrade/constants.py

@@ -11,6 +11,8 @@ DEFAULT_EXCHANGE = 'bittrex'
 PROCESS_THROTTLE_SECS = 5  # sec
 HYPEROPT_EPOCH = 100  # epochs
 RETRY_TIMEOUT = 30  # sec
+TIMEOUT_UNITS = ['minutes', 'seconds']
+EXPORT_OPTIONS = ['none', 'trades']
 DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite'
 DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite'
 UNLIMITED_STAKE_AMOUNT = 'unlimited'
@@ -60,7 +62,7 @@ DUST_PER_COIN = {
 }
 
 
-# Soure files with destination directories within user-directory
+# Source files with destination directories within user-directory
 USER_DATA_FILES = {
     'sample_strategy.py': USERPATH_STRATEGIES,
     'sample_hyperopt_advanced.py': USERPATH_HYPEROPTS,
@@ -137,7 +139,8 @@ CONF_SCHEMA = {
             'type': 'object',
             'properties': {
                 'buy': {'type': 'number', 'minimum': 1},
-                'sell': {'type': 'number', 'minimum': 1}
+                'sell': {'type': 'number', 'minimum': 1},
+                'unit': {'type': 'string', 'enum': TIMEOUT_UNITS, 'default': 'minutes'}
             }
         },
         'bid_strategy': {
@@ -258,7 +261,13 @@ CONF_SCHEMA = {
                                      'enum': TELEGRAM_SETTING_OPTIONS,
                                      'default': 'off'
                                      },
-                        'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
+                        'sell': {
+                            'type': ['string', 'object'],
+                            'additionalProperties': {
+                                'type': 'string',
+                                'enum': TELEGRAM_SETTING_OPTIONS
+                            }
+                        },
                         'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
                         'sell_fill': {
                             'type': 'string',
@@ -266,7 +275,8 @@ CONF_SCHEMA = {
                             'default': 'off'
                             },
                     }
-                }
+                },
+                'reload': {'type': 'boolean'},
             },
             'required': ['enabled', 'token', 'chat_id'],
         },
@@ -300,6 +310,7 @@ CONF_SCHEMA = {
             'required': ['enabled', 'listen_ip_address', 'listen_port', 'username', 'password']
         },
         'db_url': {'type': 'string'},
+        'export': {'type': 'string', 'enum': EXPORT_OPTIONS, 'default': 'trades'},
         'initial_state': {'type': 'string', 'enum': ['running', 'stopped']},
         'forcebuy_enable': {'type': 'boolean'},
         'disable_dataframe_checks': {'type': 'boolean'},

freqtrade/data/btanalysis.py

@@ -156,33 +156,35 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
 
         data = data['strategy'][strategy]['trades']
         df = pd.DataFrame(data)
-        df['open_date'] = pd.to_datetime(df['open_date'],
-                                         utc=True,
-                                         infer_datetime_format=True
-                                         )
-        df['close_date'] = pd.to_datetime(df['close_date'],
-                                          utc=True,
-                                          infer_datetime_format=True
-                                          )
+        if not df.empty:
+            df['open_date'] = pd.to_datetime(df['open_date'],
+                                             utc=True,
+                                             infer_datetime_format=True
+                                             )
+            df['close_date'] = pd.to_datetime(df['close_date'],
+                                              utc=True,
+                                              infer_datetime_format=True
+                                              )
     else:
         # old format - only with lists.
         df = pd.DataFrame(data, columns=BT_DATA_COLUMNS_OLD)
-
-        df['open_date'] = pd.to_datetime(df['open_date'],
-                                         unit='s',
-                                         utc=True,
-                                         infer_datetime_format=True
-                                         )
-        df['close_date'] = pd.to_datetime(df['close_date'],
-                                          unit='s',
-                                          utc=True,
-                                          infer_datetime_format=True
-                                          )
-        # Create compatibility with new format
-        df['profit_abs'] = df['close_rate'] - df['open_rate']
-    if 'profit_ratio' not in df.columns:
-        df['profit_ratio'] = df['profit_percent']
-    df = df.sort_values("open_date").reset_index(drop=True)
+        if not df.empty:
+            df['open_date'] = pd.to_datetime(df['open_date'],
+                                             unit='s',
+                                             utc=True,
+                                             infer_datetime_format=True
+                                             )
+            df['close_date'] = pd.to_datetime(df['close_date'],
+                                              unit='s',
+                                              utc=True,
+                                              infer_datetime_format=True
+                                              )
+            # Create compatibility with new format
+            df['profit_abs'] = df['close_rate'] - df['open_rate']
+    if not df.empty:
+        if 'profit_ratio' not in df.columns:
+            df['profit_ratio'] = df['profit_percent']
+        df = df.sort_values("open_date").reset_index(drop=True)
     return df
 
 

freqtrade/data/converter.py

@@ -49,7 +49,7 @@ def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *,
                           fill_missing: bool = True,
                           drop_incomplete: bool = True) -> DataFrame:
     """
-    Clense a OHLCV dataframe by
+    Cleanse a OHLCV dataframe by
       * Grouping it by date (removes duplicate tics)
       * dropping last candles if requested
       * Filling up missing data (if requested)
@@ -145,6 +145,27 @@ def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date',
     return df
 
 
+def trim_dataframes(preprocessed: Dict[str, DataFrame], timerange,
+                    startup_candles: int) -> Dict[str, DataFrame]:
+    """
+    Trim startup period from analyzed dataframes
+    :param preprocessed: Dict of pair: dataframe
+    :param timerange: timerange (use start and end date if available)
+    :param startup_candles: Startup-candles that should be removed
+    :return: Dict of trimmed dataframes
+    """
+    processed: Dict[str, DataFrame] = {}
+
+    for pair, df in preprocessed.items():
+        trimed_df = trim_dataframe(df, timerange, startup_candles=startup_candles)
+        if not trimed_df.empty:
+            processed[pair] = trimed_df
+        else:
+            logger.warning(f'{pair} has no data left after adjusting for startup candles, '
+                           f'skipping.')
+    return processed
+
+
 def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
     """
     TODO: This should get a dedicated test

freqtrade/data/dataprovider.py

@@ -12,21 +12,32 @@ from pandas import DataFrame
 
 from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe
 from freqtrade.data.history import load_pair_history
+from freqtrade.enums import RunMode
 from freqtrade.exceptions import ExchangeError, OperationalException
 from freqtrade.exchange import Exchange
-from freqtrade.state import RunMode
 
 
 logger = logging.getLogger(__name__)
 
+NO_EXCHANGE_EXCEPTION = 'Exchange is not available to DataProvider.'
+MAX_DATAFRAME_CANDLES = 1000
+
 
 class DataProvider:
 
-    def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None:
+    def __init__(self, config: dict, exchange: Optional[Exchange], pairlists=None) -> None:
         self._config = config
         self._exchange = exchange
         self._pairlists = pairlists
         self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
+        self.__slice_index: Optional[int] = None
+
+    def _set_dataframe_max_index(self, limit_index: int):
+        """
+        Limit analyzed dataframe to max specified index.
+        :param limit_index: dataframe index.
+        """
+        self.__slice_index = limit_index
 
     def _set_cached_df(self, pair: str, timeframe: str, dataframe: DataFrame) -> None:
         """
@@ -45,40 +56,6 @@ class DataProvider:
         """
         self._pairlists = pairlists
 
-    def refresh(self,
-                pairlist: ListPairsWithTimeframes,
-                helping_pairs: ListPairsWithTimeframes = None) -> None:
-        """
-        Refresh data, called with each cycle
-        """
-        if helping_pairs:
-            self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs)
-        else:
-            self._exchange.refresh_latest_ohlcv(pairlist)
-
-    @property
-    def available_pairs(self) -> ListPairsWithTimeframes:
-        """
-        Return a list of tuples containing (pair, timeframe) for which data is currently cached.
-        Should be whitelist + open trades.
-        """
-        return list(self._exchange._klines.keys())
-
-    def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame:
-        """
-        Get candle (OHLCV) data for the given pair as DataFrame
-        Please use the `available_pairs` method to verify which pairs are currently cached.
-        :param pair: pair to get the data for
-        :param timeframe: Timeframe to get data for
-        :param copy: copy dataframe before returning if True.
-                     Use False only for read-only operations (where the dataframe is not modified)
-        """
-        if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
-            return self._exchange.klines((pair, timeframe or self._config['timeframe']),
-                                         copy=copy)
-        else:
-            return DataFrame()
-
     def historic_ohlcv(self, pair: str, timeframe: str = None) -> DataFrame:
         """
         Get stored historical candle (OHLCV) data
@@ -111,47 +88,27 @@ class DataProvider:
 
     def get_analyzed_dataframe(self, pair: str, timeframe: str) -> Tuple[DataFrame, datetime]:
         """
+        Retrieve the analyzed dataframe. Returns the full dataframe in trade mode (live / dry),
+        and the last 1000 candles (up to the time evaluated at this moment) in all other modes.
         :param pair: pair to get the data for
         :param timeframe: timeframe to get data for
         :return: Tuple of (Analyzed Dataframe, lastrefreshed) for the requested pair / timeframe
             combination.
             Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached.
         """
-        if (pair, timeframe) in self.__cached_pairs:
-            return self.__cached_pairs[(pair, timeframe)]
+        pair_key = (pair, timeframe)
+        if pair_key in self.__cached_pairs:
+            if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
+                df, date = self.__cached_pairs[pair_key]
+            else:
+                df, date = self.__cached_pairs[pair_key]
+                if self.__slice_index is not None:
+                    max_index = self.__slice_index
+                    df = df.iloc[max(0, max_index - MAX_DATAFRAME_CANDLES):max_index]
+            return df, date
         else:
-
             return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc))
 
-    def market(self, pair: str) -> Optional[Dict[str, Any]]:
-        """
-        Return market data for the pair
-        :param pair: Pair to get the data for
-        :return: Market data dict from ccxt or None if market info is not available for the pair
-        """
-        return self._exchange.markets.get(pair)
-
-    def ticker(self, pair: str):
-        """
-        Return last ticker data from exchange
-        :param pair: Pair to get the data for
-        :return: Ticker dict from exchange or empty dict if ticker is not available for the pair
-        """
-        try:
-            return self._exchange.fetch_ticker(pair)
-        except ExchangeError:
-            return {}
-
-    def orderbook(self, pair: str, maximum: int) -> Dict[str, List]:
-        """
-        Fetch latest l2 orderbook data
-        Warning: Does a network request - so use with common sense.
-        :param pair: pair to get the data for
-        :param maximum: Maximum number of orderbook entries to query
-        :return: dict including bids/asks with a total of `maximum` entries.
-        """
-        return self._exchange.fetch_l2_order_book(pair, maximum)
-
     @property
     def runmode(self) -> RunMode:
         """
@@ -173,3 +130,86 @@ class DataProvider:
             return self._pairlists.whitelist.copy()
         else:
             raise OperationalException("Dataprovider was not initialized with a pairlist provider.")
+
+    def clear_cache(self):
+        """
+        Clear pair dataframe cache.
+        """
+        self.__cached_pairs = {}
+
+    # Exchange functions
+
+    def refresh(self,
+                pairlist: ListPairsWithTimeframes,
+                helping_pairs: ListPairsWithTimeframes = None) -> None:
+        """
+        Refresh data, called with each cycle
+        """
+        if self._exchange is None:
+            raise OperationalException(NO_EXCHANGE_EXCEPTION)
+        if helping_pairs:
+            self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs)
+        else:
+            self._exchange.refresh_latest_ohlcv(pairlist)
+
+    @property
+    def available_pairs(self) -> ListPairsWithTimeframes:
+        """
+        Return a list of tuples containing (pair, timeframe) for which data is currently cached.
+        Should be whitelist + open trades.
+        """
+        if self._exchange is None:
+            raise OperationalException(NO_EXCHANGE_EXCEPTION)
+        return list(self._exchange._klines.keys())
+
+    def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame:
+        """
+        Get candle (OHLCV) data for the given pair as DataFrame
+        Please use the `available_pairs` method to verify which pairs are currently cached.
+        :param pair: pair to get the data for
+        :param timeframe: Timeframe to get data for
+        :param copy: copy dataframe before returning if True.
+                     Use False only for read-only operations (where the dataframe is not modified)
+        """
+        if self._exchange is None:
+            raise OperationalException(NO_EXCHANGE_EXCEPTION)
+        if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
+            return self._exchange.klines((pair, timeframe or self._config['timeframe']),
+                                         copy=copy)
+        else:
+            return DataFrame()
+
+    def market(self, pair: str) -> Optional[Dict[str, Any]]:
+        """
+        Return market data for the pair
+        :param pair: Pair to get the data for
+        :return: Market data dict from ccxt or None if market info is not available for the pair
+        """
+        if self._exchange is None:
+            raise OperationalException(NO_EXCHANGE_EXCEPTION)
+        return self._exchange.markets.get(pair)
+
+    def ticker(self, pair: str):
+        """
+        Return last ticker data from exchange
+        :param pair: Pair to get the data for
+        :return: Ticker dict from exchange or empty dict if ticker is not available for the pair
+        """
+        if self._exchange is None:
+            raise OperationalException(NO_EXCHANGE_EXCEPTION)
+        try:
+            return self._exchange.fetch_ticker(pair)
+        except ExchangeError:
+            return {}
+
+    def orderbook(self, pair: str, maximum: int) -> Dict[str, List]:
+        """
+        Fetch latest l2 orderbook data
+        Warning: Does a network request - so use with common sense.
+        :param pair: pair to get the data for
+        :param maximum: Maximum number of orderbook entries to query
+        :return: dict including bids/asks with a total of `maximum` entries.
+        """
+        if self._exchange is None:
+            raise OperationalException(NO_EXCHANGE_EXCEPTION)
+        return self._exchange.fetch_l2_order_book(pair, maximum)

freqtrade/data/history/hdf5datahandler.py

@@ -52,8 +52,8 @@ class HDF5DataHandler(IDataHandler):
         """
         Store data in hdf5 file.
         :param pair: Pair - used to generate filename
-        :timeframe: Timeframe - used to generate filename
-        :data: Dataframe containing OHLCV data
+        :param timeframe: Timeframe - used to generate filename
+        :param data: Dataframe containing OHLCV data
         :return: None
         """
         key = self._pair_ohlcv_key(pair, timeframe)

freqtrade/data/history/history_utils.py

@@ -113,6 +113,7 @@ def refresh_data(datadir: Path,
     :param timeframe: Timeframe (e.g. "5m")
     :param pairs: List of pairs to load
     :param exchange: Exchange object
+    :param data_format: dataformat to use
     :param timerange: Limit data to be loaded to this timerange
     """
     data_handler = get_datahandler(datadir, data_format)
@@ -265,9 +266,13 @@ def _download_trades_history(exchange: Exchange,
     """
     try:
 
-        since = timerange.startts * 1000 if \
-            (timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift(
-                days=-new_pairs_days).float_timestamp) * 1000
+        until = None
+        if (timerange and timerange.starttype == 'date'):
+            since = timerange.startts * 1000
+            if timerange.stoptype == 'date':
+                until = timerange.stopts * 1000
+        else:
+            since = int(arrow.utcnow().shift(days=-new_pairs_days).float_timestamp) * 1000
 
         trades = data_handler.trades_load(pair)
 
@@ -295,6 +300,7 @@ def _download_trades_history(exchange: Exchange,
         # Default since_ms to 30 days if nothing is given
         new_trades = exchange.get_historic_trades(pair=pair,
                                                   since=since,
+                                                  until=until,
                                                   from_id=from_id,
                                                   )
         trades.extend(new_trades[1])
@@ -367,7 +373,7 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
                 logger.exception(f'Could not convert {pair} to OHLCV.')
 
 
-def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
+def get_timerange(data: Dict[str, DataFrame]) -> Tuple[datetime, datetime]:
     """
     Get the maximum common timerange for the given backtest data.
 
@@ -375,7 +381,7 @@ def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]
     :return: tuple containing min_date, max_date
     """
     timeranges = [
-        (arrow.get(frame['date'].min()), arrow.get(frame['date'].max()))
+        (frame['date'].min().to_pydatetime(), frame['date'].max().to_pydatetime())
         for frame in data.values()
     ]
     return (min(timeranges, key=operator.itemgetter(0))[0],

freqtrade/data/history/idatahandler.py

@@ -49,8 +49,8 @@ class IDataHandler(ABC):
         """
         Store ohlcv data.
         :param pair: Pair - used to generate filename
-        :timeframe: Timeframe - used to generate filename
-        :data: Dataframe containing OHLCV data
+        :param timeframe: Timeframe - used to generate filename
+        :param data: Dataframe containing OHLCV data
         :return: None
         """
 
@@ -245,8 +245,8 @@ def get_datahandler(datadir: Path, data_format: str = None,
                     data_handler: IDataHandler = None) -> IDataHandler:
     """
     :param datadir: Folder to save data
-    :data_format: dataformat to use
-    :data_handler: returns this datahandler if it exists or initializes a new one
+    :param data_format: dataformat to use
+    :param data_handler: returns this datahandler if it exists or initializes a new one
     """
 
     if not data_handler:

freqtrade/data/history/jsondatahandler.py

@@ -55,8 +55,8 @@ class JsonDataHandler(IDataHandler):
             format looks as follows:
             [[<date>,<open>,<high>,<low>,<close>]]
         :param pair: Pair - used to generate filename
-        :timeframe: Timeframe - used to generate filename
-        :data: Dataframe containing OHLCV data
+        :param timeframe: Timeframe - used to generate filename
+        :param data: Dataframe containing OHLCV data
         :return: None
         """
         filename = self._pair_data_filename(self._datadir, pair, timeframe)

freqtrade/edge/edge_positioning.py

@@ -1,6 +1,8 @@
 # pragma pylint: disable=W0603
 """ Edge positioning package """
 import logging
+from collections import defaultdict
+from copy import deepcopy
 from typing import Any, Dict, List, NamedTuple
 
 import arrow
@@ -11,9 +13,11 @@ from pandas import DataFrame
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT
 from freqtrade.data.history import get_timerange, load_data, refresh_data
+from freqtrade.enums import RunMode, SellType
 from freqtrade.exceptions import OperationalException
+from freqtrade.exchange.exchange import timeframe_to_seconds
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
-from freqtrade.strategy.interface import SellType
+from freqtrade.strategy.interface import IStrategy
 
 
 logger = logging.getLogger(__name__)
@@ -45,7 +49,7 @@ class Edge:
 
         self.config = config
         self.exchange = exchange
-        self.strategy = strategy
+        self.strategy: IStrategy = strategy
 
         self.edge_config = self.config.get('edge', {})
         self._cached_pairs: Dict[str, Any] = {}  # Keeps a list of pairs
@@ -102,14 +106,33 @@ class Edge:
         logger.info('Using local backtesting data (using whitelist in given config) ...')
 
         if self._refresh_pairs:
+            timerange_startup = deepcopy(self._timerange)
+            timerange_startup.subtract_start(timeframe_to_seconds(
+                self.strategy.timeframe) * self.strategy.startup_candle_count)
             refresh_data(
                 datadir=self.config['datadir'],
                 pairs=pairs,
                 exchange=self.exchange,
                 timeframe=self.strategy.timeframe,
-                timerange=self._timerange,
+                timerange=timerange_startup,
                 data_format=self.config.get('dataformat_ohlcv', 'json'),
             )
+            # Download informative pairs too
+            res = defaultdict(list)
+            for p, t in self.strategy.informative_pairs():
+                res[t].append(p)
+            for timeframe, inf_pairs in res.items():
+                timerange_startup = deepcopy(self._timerange)
+                timerange_startup.subtract_start(timeframe_to_seconds(
+                    timeframe) * self.strategy.startup_candle_count)
+                refresh_data(
+                    datadir=self.config['datadir'],
+                    pairs=inf_pairs,
+                    exchange=self.exchange,
+                    timeframe=timeframe,
+                    timerange=timerange_startup,
+                    data_format=self.config.get('dataformat_ohlcv', 'json'),
+                )
 
         data = load_data(
             datadir=self.config['datadir'],
@@ -125,8 +148,11 @@ class Edge:
             self._cached_pairs = {}
             logger.critical("No data found. Edge is stopped ...")
             return False
-
+        # Fake run-mode to Edge
+        prior_rm = self.config['runmode']
+        self.config['runmode'] = RunMode.EDGE
         preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
+        self.config['runmode'] = prior_rm
 
         # Print timeframe
         min_date, max_date = get_timerange(preprocessed)
@@ -183,7 +209,7 @@ class Edge:
         if pair in self._cached_pairs:
             return self._cached_pairs[pair].stoploss
         else:
-            logger.warning('tried to access stoploss of a non-existing pair, '
+            logger.warning(f'Tried to access stoploss of non-existing pair {pair}, '
                            'strategy stoploss is returned instead.')
             return self.strategy.stoploss
 
@@ -214,7 +240,7 @@ class Edge:
 
         return self._final_pairs
 
-    def accepted_pairs(self) -> list:
+    def accepted_pairs(self) -> List[Dict[str, Any]]:
         """
         return a list of accepted pairs along with their winrate, expectancy and stoploss
         """
@@ -275,7 +301,7 @@ class Edge:
     def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]:
         """
         This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs
-        The calulation will be done per pair and per strategy.
+        The calculation will be done per pair and per strategy.
         """
         # Removing pairs having less than min_trades_number
         min_trades_number = self.edge_config.get('min_trade_number', 10)

freqtrade/enums/__init__.py

@@ -0,0 +1,6 @@
+# flake8: noqa: F401
+from freqtrade.enums.rpcmessagetype import RPCMessageType
+from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode
+from freqtrade.enums.selltype import SellType
+from freqtrade.enums.signaltype import SignalType
+from freqtrade.enums.state import State

freqtrade/enums/rpcmessagetype.py

@@ -0,0 +1,19 @@
+from enum import Enum
+
+
+class RPCMessageType(Enum):
+    STATUS = 'status'
+    WARNING = 'warning'
+    STARTUP = 'startup'
+    BUY = 'buy'
+    BUY_FILL = 'buy_fill'
+    BUY_CANCEL = 'buy_cancel'
+    SELL = 'sell'
+    SELL_FILL = 'sell_fill'
+    SELL_CANCEL = 'sell_cancel'
+
+    def __repr__(self):
+        return self.value
+
+    def __str__(self):
+        return self.value

freqtrade/enums/runmode.py

@@ -1,23 +1,6 @@
-# pragma pylint: disable=too-few-public-methods
-
-"""
-Bot state constant
-"""
 from enum import Enum
 
 
-class State(Enum):
-    """
-    Bot application states
-    """
-    RUNNING = 1
-    STOPPED = 2
-    RELOAD_CONFIG = 3
-
-    def __str__(self):
-        return f"{self.name.lower()}"
-
-
 class RunMode(Enum):
     """
     Bot running mode (backtest, hyperopt, ...)

freqtrade/enums/selltype.py

@@ -0,0 +1,20 @@
+from enum import Enum
+
+
+class SellType(Enum):
+    """
+    Enum to distinguish between sell reasons
+    """
+    ROI = "roi"
+    STOP_LOSS = "stop_loss"
+    STOPLOSS_ON_EXCHANGE = "stoploss_on_exchange"
+    TRAILING_STOP_LOSS = "trailing_stop_loss"
+    SELL_SIGNAL = "sell_signal"
+    FORCE_SELL = "force_sell"
+    EMERGENCY_SELL = "emergency_sell"
+    CUSTOM_SELL = "custom_sell"
+    NONE = ""
+
+    def __str__(self):
+        # explicitly convert to String to help with exporting data.
+        return self.value

freqtrade/enums/signaltype.py

@@ -0,0 +1,9 @@
+from enum import Enum
+
+
+class SignalType(Enum):
+    """
+    Enum to distinguish between buy and sell signals
+    """
+    BUY = "buy"
+    SELL = "sell"

freqtrade/enums/state.py

@@ -0,0 +1,13 @@
+from enum import Enum
+
+
+class State(Enum):
+    """
+    Bot application states
+    """
+    RUNNING = 1
+    STOPPED = 2
+    RELOAD_CONFIG = 3
+
+    def __str__(self):
+        return f"{self.name.lower()}"

freqtrade/exceptions.py

@@ -47,7 +47,7 @@ class InvalidOrderException(ExchangeError):
 class RetryableOrderError(InvalidOrderException):
     """
     This is returned when the order is not found.
-    This Error will be repeated with increasing backof (in line with DDosError).
+    This Error will be repeated with increasing backoff (in line with DDosError).
     """
 
 
@@ -75,6 +75,6 @@ class DDosProtection(TemporaryError):
 
 class StrategyError(FreqtradeException):
     """
-    Errors with custom user-code deteced.
+    Errors with custom user-code detected.
     Usually caused by errors in the strategy.
     """

freqtrade/exchange/__init__.py

@@ -7,6 +7,7 @@ from freqtrade.exchange.bibox import Bibox
 from freqtrade.exchange.binance import Binance
 from freqtrade.exchange.bittrex import Bittrex
 from freqtrade.exchange.bybit import Bybit
+from freqtrade.exchange.coinbasepro import Coinbasepro
 from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges,
                                          is_exchange_known_ccxt, is_exchange_officially_supported,
                                          market_is_active, timeframe_to_minutes, timeframe_to_msecs,
@@ -14,5 +15,6 @@ from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges,
                                          timeframe_to_seconds, validate_exchange,
                                          validate_exchanges)
 from freqtrade.exchange.ftx import Ftx
+from freqtrade.exchange.hitbtc import Hitbtc
 from freqtrade.exchange.kraken import Kraken
 from freqtrade.exchange.kucoin import Kucoin

freqtrade/exchange/binance.py

@@ -68,6 +68,7 @@ class Binance(Exchange):
                                            amount=amount, price=rate, params=params)
             logger.info('stoploss limit order added for %s. '
                         'stop price: %s. limit: %s', pair, stop_price, rate)
+            self._log_exchange_response('create_stoploss_order', order)
             return order
         except ccxt.InsufficientFunds as e:
             raise InsufficientFundsError(

freqtrade/exchange/bybit.py

@@ -18,7 +18,6 @@ class Bybit(Exchange):
     may still not work as expected.
     """
 
-    # fetchCurrencies API point requires authentication for Bybit,
     _ft_has: Dict = {
         "ohlcv_candle_limit": 200,
     }

freqtrade/exchange/coinbasepro.py

@@ -0,0 +1,23 @@
+""" CoinbasePro exchange subclass """
+import logging
+from typing import Dict
+
+from freqtrade.exchange import Exchange
+
+
+logger = logging.getLogger(__name__)
+
+
+class Coinbasepro(Exchange):
+    """
+    CoinbasePro exchange class. Contains adjustments needed for Freqtrade to work
+    with this exchange.
+
+    Please note that this exchange is not included in the list of exchanges
+    officially supported by the Freqtrade development team. So some features
+    may still not work as expected.
+    """
+
+    _ft_has: Dict = {
+        "ohlcv_candle_limit": 300,
+    }

freqtrade/exchange/exchange.py

@@ -22,8 +22,8 @@ from pandas import DataFrame
 from freqtrade.constants import DEFAULT_AMOUNT_RESERVE_PERCENT, ListPairsWithTimeframes
 from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
 from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
-                                  InvalidOrderException, OperationalException, RetryableOrderError,
-                                  TemporaryError)
+                                  InvalidOrderException, OperationalException, PricingError,
+                                  RetryableOrderError, TemporaryError)
 from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES,
                                        EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier,
                                        retrier_async)
@@ -59,6 +59,7 @@ class Exchange:
     _ft_has_default: Dict = {
         "stoploss_on_exchange": False,
         "order_time_in_force": ["gtc"],
+        "ohlcv_params": {},
         "ohlcv_candle_limit": 500,
         "ohlcv_partial_candle": True,
         "trades_pagination": "time",  # Possible are "time" or "id"
@@ -87,6 +88,11 @@ class Exchange:
 
         # Cache for 10 minutes ...
         self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=1, ttl=60 * 10)
+        # Cache values for 1800 to avoid frequent polling of the exchange for prices
+        # Caching only applies to RPC methods, so prices for open trades are still
+        # refreshed once every iteration.
+        self._sell_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800)
+        self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800)
 
         # Holds candles
         self._klines: Dict[Tuple[str, str], DataFrame] = {}
@@ -98,6 +104,7 @@ class Exchange:
             logger.info('Instance is running with dry_run enabled')
         logger.info(f"Using CCXT {ccxt.__version__}")
         exchange_config = config['exchange']
+        self.log_responses = exchange_config.get('log_responses', False)
 
         # Deep merge ft_has with default ft_has options
         self._ft_has = deep_merge_dicts(self._ft_has, deepcopy(self._ft_has_default))
@@ -220,10 +227,15 @@ class Exchange:
         """exchange ccxt precisionMode"""
         return self._api.precisionMode
 
+    def _log_exchange_response(self, endpoint, response) -> None:
+        """ Log exchange responses """
+        if self.log_responses:
+            logger.info(f"API {endpoint}: {response}")
+
     def ohlcv_candle_limit(self, timeframe: str) -> int:
         """
         Exchange ohlcv candle limit
-        Uses ohlcv_candle_limit_per_timeframe if the exchange has different limts
+        Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits
         per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit
         :param timeframe: Timeframe to check
         :return: Candle limit as integer
@@ -463,11 +475,11 @@ class Exchange:
         return endpoint in self._api.has and self._api.has[endpoint]
 
     def amount_to_precision(self, pair: str, amount: float) -> float:
-        '''
+        """
         Returns the amount to buy or sell to a precision the Exchange accepts
-        Reimplementation of ccxt internal methods - ensuring we can test the result is correct
+        Re-implementation of ccxt internal methods - ensuring we can test the result is correct
         based on our definitions.
-        '''
+        """
         if self.markets[pair]['precision']['amount']:
             amount = float(decimal_to_precision(amount, rounding_mode=TRUNCATE,
                                                 precision=self.markets[pair]['precision']['amount'],
@@ -477,14 +489,14 @@ class Exchange:
         return amount
 
     def price_to_precision(self, pair: str, price: float) -> float:
-        '''
+        """
         Returns the price rounded up to the precision the Exchange accepts.
-        Partial Reimplementation of ccxt internal method decimal_to_precision(),
+        Partial Re-implementation of ccxt internal method decimal_to_precision(),
         which does not support rounding up
         TODO: If ccxt supports ROUND_UP for decimal_to_precision(), we could remove this and
         align with amount_to_precision().
         Rounds up
-        '''
+        """
         if self.markets[pair]['precision']['price']:
             # price = float(decimal_to_precision(price, rounding_mode=ROUND,
             #                                    precision=self.markets[pair]['precision']['price'],
@@ -549,11 +561,13 @@ class Exchange:
         # See also #2575 at github.
         return max(min_stake_amounts) * amount_reserve_percent
 
+    # Dry-run methods
+
     def create_dry_run_order(self, pair: str, ordertype: str, side: str, amount: float,
                              rate: float, params: Dict = {}) -> Dict[str, Any]:
         order_id = f'dry_run_{side}_{datetime.now().timestamp()}'
         _amount = self.amount_to_precision(pair, amount)
-        dry_order = {
+        dry_order: Dict[str, Any] = {
             'id': order_id,
             'symbol': pair,
             'price': rate,
@@ -569,26 +583,110 @@ class Exchange:
             'fee': None,
             'info': {}
         }
-        self._store_dry_order(dry_order, pair)
+        if dry_order["type"] in ["stop_loss_limit", "stop-loss-limit"]:
+            dry_order["info"] = {"stopPrice": dry_order["price"]}
+
+        if dry_order["type"] == "market":
+            # Update market order pricing
+            average = self.get_dry_market_fill_price(pair, side, amount, rate)
+            dry_order.update({
+                'average': average,
+                'cost': dry_order['amount'] * average,
+            })
+            dry_order = self.add_dry_order_fee(pair, dry_order)
+
+        dry_order = self.check_dry_limit_order_filled(dry_order)
+
+        self._dry_run_open_orders[dry_order["id"]] = dry_order
         # Copy order and close it - so the returned order is open unless it's a market order
         return dry_order
 
-    def _store_dry_order(self, dry_order: Dict, pair: str) -> None:
-        closed_order = dry_order.copy()
-        if closed_order['type'] in ["market", "limit"]:
-            closed_order.update({
-                'status': 'closed',
-                'filled': closed_order['amount'],
-                'remaining': 0,
-                'fee': {
-                    'currency': self.get_pair_quote_currency(pair),
-                    'cost': dry_order['cost'] * self.get_fee(pair),
-                    'rate': self.get_fee(pair)
-                }
-            })
-        if closed_order["type"] in ["stop_loss_limit", "stop-loss-limit"]:
-            closed_order["info"].update({"stopPrice": closed_order["price"]})
-        self._dry_run_open_orders[closed_order["id"]] = closed_order
+    def add_dry_order_fee(self, pair: str, dry_order: Dict[str, Any]) -> Dict[str, Any]:
+        dry_order.update({
+            'fee': {
+                'currency': self.get_pair_quote_currency(pair),
+                'cost': dry_order['cost'] * self.get_fee(pair),
+                'rate': self.get_fee(pair)
+            }
+        })
+        return dry_order
+
+    def get_dry_market_fill_price(self, pair: str, side: str, amount: float, rate: float) -> float:
+        """
+        Get the market order fill price based on orderbook interpolation
+        """
+        if self.exchange_has('fetchL2OrderBook'):
+            ob = self.fetch_l2_order_book(pair, 20)
+            ob_type = 'asks' if side == 'buy' else 'bids'
+
+            remaining_amount = amount
+            filled_amount = 0
+            for book_entry in ob[ob_type]:
+                book_entry_price = book_entry[0]
+                book_entry_coin_volume = book_entry[1]
+                if remaining_amount > 0:
+                    if remaining_amount < book_entry_coin_volume:
+                        filled_amount += remaining_amount * book_entry_price
+                    else:
+                        filled_amount += book_entry_coin_volume * book_entry_price
+                    remaining_amount -= book_entry_coin_volume
+                else:
+                    break
+            else:
+                # If remaining_amount wasn't consumed completely (break was not called)
+                filled_amount += remaining_amount * book_entry_price
+            forecast_avg_filled_price = filled_amount / amount
+            return self.price_to_precision(pair, forecast_avg_filled_price)
+
+        return rate
+
+    def _is_dry_limit_order_filled(self, pair: str, side: str, limit: float) -> bool:
+        if not self.exchange_has('fetchL2OrderBook'):
+            return True
+        ob = self.fetch_l2_order_book(pair, 1)
+        if side == 'buy':
+            price = ob['asks'][0][0]
+            logger.debug(f"{pair} checking dry buy-order: price={price}, limit={limit}")
+            if limit >= price:
+                return True
+        else:
+            price = ob['bids'][0][0]
+            logger.debug(f"{pair} checking dry sell-order: price={price}, limit={limit}")
+            if limit <= price:
+                return True
+        return False
+
+    def check_dry_limit_order_filled(self, order: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Check dry-run limit order fill and update fee (if it filled).
+        """
+        if order['status'] != "closed" and order['type'] in ["limit"]:
+            pair = order['symbol']
+            if self._is_dry_limit_order_filled(pair, order['side'], order['price']):
+                order.update({
+                    'status': 'closed',
+                    'filled': order['amount'],
+                    'remaining': 0,
+                })
+                self.add_dry_order_fee(pair, order)
+
+        return order
+
+    def fetch_dry_run_order(self, order_id) -> Dict[str, Any]:
+        """
+        Return dry-run order
+        Only call if running in dry-run mode.
+        """
+        try:
+            order = self._dry_run_open_orders[order_id]
+            order = self.check_dry_limit_order_filled(order)
+            return order
+        except KeyError as e:
+            # Gracefully handle errors with dry-run orders.
+            raise InvalidOrderException(
+                f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e
+
+    # Order handling
 
     def create_order(self, pair: str, ordertype: str, side: str, amount: float,
                      rate: float, params: Dict = {}) -> Dict:
@@ -599,8 +697,10 @@ class Exchange:
                            or self._api.options.get("createMarketBuyOrderRequiresPrice", False))
             rate_for_order = self.price_to_precision(pair, rate) if needs_price else None
 
-            return self._api.create_order(pair, ordertype, side,
-                                          amount, rate_for_order, params)
+            order = self._api.create_order(pair, ordertype, side,
+                                           amount, rate_for_order, params)
+            self._log_exchange_response('create_order', order)
+            return order
 
         except ccxt.InsufficientFunds as e:
             raise InsufficientFundsError(
@@ -666,6 +766,134 @@ class Exchange:
 
         raise OperationalException(f"stoploss is not implemented for {self.name}.")
 
+    @retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
+    def fetch_order(self, order_id: str, pair: str) -> Dict:
+        if self._config['dry_run']:
+            return self.fetch_dry_run_order(order_id)
+        try:
+            order = self._api.fetch_order(order_id, pair)
+            self._log_exchange_response('fetch_order', order)
+            return order
+        except ccxt.OrderNotFound as e:
+            raise RetryableOrderError(
+                f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e
+        except ccxt.InvalidOrder as e:
+            raise InvalidOrderException(
+                f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(
+                f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e
+        except ccxt.BaseError as e:
+            raise OperationalException(e) from e
+
+    # Assign method to fetch_stoploss_order to allow easy overriding in other classes
+    fetch_stoploss_order = fetch_order
+
+    def fetch_order_or_stoploss_order(self, order_id: str, pair: str,
+                                      stoploss_order: bool = False) -> Dict:
+        """
+        Simple wrapper calling either fetch_order or fetch_stoploss_order depending on
+        the stoploss_order parameter
+        :param order_id: OrderId to fetch order
+        :param pair: Pair corresponding to order_id
+        :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order.
+        """
+        if stoploss_order:
+            return self.fetch_stoploss_order(order_id, pair)
+        return self.fetch_order(order_id, pair)
+
+    def check_order_canceled_empty(self, order: Dict) -> bool:
+        """
+        Verify if an order has been cancelled without being partially filled
+        :param order: Order dict as returned from fetch_order()
+        :return: True if order has been cancelled without being filled, False otherwise.
+        """
+        return (order.get('status') in ('closed', 'canceled', 'cancelled')
+                and order.get('filled') == 0.0)
+
+    @retrier
+    def cancel_order(self, order_id: str, pair: str) -> Dict:
+        if self._config['dry_run']:
+            try:
+                order = self.fetch_dry_run_order(order_id)
+
+                order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']})
+                return order
+            except InvalidOrderException:
+                return {}
+
+        try:
+            order = self._api.cancel_order(order_id, pair)
+            self._log_exchange_response('cancel_order', order)
+            return order
+        except ccxt.InvalidOrder as e:
+            raise InvalidOrderException(
+                f'Could not cancel order. Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(
+                f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e
+        except ccxt.BaseError as e:
+            raise OperationalException(e) from e
+
+    # Assign method to cancel_stoploss_order to allow easy overriding in other classes
+    cancel_stoploss_order = cancel_order
+
+    def is_cancel_order_result_suitable(self, corder) -> bool:
+        if not isinstance(corder, dict):
+            return False
+
+        required = ('fee', 'status', 'amount')
+        return all(k in corder for k in required)
+
+    def cancel_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict:
+        """
+        Cancel order returning a result.
+        Creates a fake result if cancel order returns a non-usable result
+        and fetch_order does not work (certain exchanges don't return cancelled orders)
+        :param order_id: Orderid to cancel
+        :param pair: Pair corresponding to order_id
+        :param amount: Amount to use for fake response
+        :return: Result from either cancel_order if usable, or fetch_order
+        """
+        try:
+            corder = self.cancel_order(order_id, pair)
+            if self.is_cancel_order_result_suitable(corder):
+                return corder
+        except InvalidOrderException:
+            logger.warning(f"Could not cancel order {order_id} for {pair}.")
+        try:
+            order = self.fetch_order(order_id, pair)
+        except InvalidOrderException:
+            logger.warning(f"Could not fetch cancelled order {order_id}.")
+            order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
+
+        return order
+
+    def cancel_stoploss_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict:
+        """
+        Cancel stoploss order returning a result.
+        Creates a fake result if cancel order returns a non-usable result
+        and fetch_order does not work (certain exchanges don't return cancelled orders)
+        :param order_id: stoploss-order-id to cancel
+        :param pair: Pair corresponding to order_id
+        :param amount: Amount to use for fake response
+        :return: Result from either cancel_order if usable, or fetch_order
+        """
+        corder = self.cancel_stoploss_order(order_id, pair)
+        if self.is_cancel_order_result_suitable(corder):
+            return corder
+        try:
+            order = self.fetch_stoploss_order(order_id, pair)
+        except InvalidOrderException:
+            logger.warning(f"Could not fetch cancelled stoploss order {order_id}.")
+            order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
+
+        return order
+
     @retrier
     def get_balances(self) -> dict:
 
@@ -712,6 +940,8 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
+    # Pricing info
+
     @retrier
     def fetch_ticker(self, pair: str) -> dict:
         try:
@@ -728,6 +958,265 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
+    @staticmethod
+    def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]],
+                               range_required: bool = True):
+        """
+        Get next greater value in the list.
+        Used by fetch_l2_order_book if the api only supports a limited range
+        """
+        if not limit_range:
+            return limit
+
+        result = min([x for x in limit_range if limit <= x] + [max(limit_range)])
+        if not range_required and limit > result:
+            # Range is not required - we can use None as parameter.
+            return None
+        return result
+
+    @retrier
+    def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
+        """
+        Get L2 order book from exchange.
+        Can be limited to a certain amount (if supported).
+        Returns a dict in the format
+        {'asks': [price, volume], 'bids': [price, volume]}
+        """
+        limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'],
+                                             self._ft_has['l2_limit_range_required'])
+        try:
+
+            return self._api.fetch_l2_order_book(pair, limit1)
+        except ccxt.NotSupported as e:
+            raise OperationalException(
+                f'Exchange {self._api.name} does not support fetching order book.'
+                f'Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(
+                f'Could not get order book due to {e.__class__.__name__}. Message: {e}') from e
+        except ccxt.BaseError as e:
+            raise OperationalException(e) from e
+
+    def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1,
+                        order_book_min: int = 1):
+        """
+        Helper generator to query orderbook in loop (used for early sell-order placing)
+        """
+        order_book = self.fetch_l2_order_book(pair, order_book_max)
+        for i in range(order_book_min, order_book_max + 1):
+            yield order_book[side][i - 1][0]
+
+    def get_buy_rate(self, pair: str, refresh: bool) -> float:
+        """
+        Calculates bid target between current ask price and last price
+        :param pair: Pair to get rate for
+        :param refresh: allow cached data
+        :return: float: Price
+        :raises PricingError if orderbook price could not be determined.
+        """
+        if not refresh:
+            rate = self._buy_rate_cache.get(pair)
+            # Check if cache has been invalidated
+            if rate:
+                logger.debug(f"Using cached buy rate for {pair}.")
+                return rate
+
+        bid_strategy = self._config.get('bid_strategy', {})
+        if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False):
+
+            order_book_top = bid_strategy.get('order_book_top', 1)
+            order_book = self.fetch_l2_order_book(pair, order_book_top)
+            logger.debug('order_book %s', order_book)
+            # top 1 = index 0
+            try:
+                rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0]
+            except (IndexError, KeyError) as e:
+                logger.warning(
+                    "Buy Price from orderbook could not be determined."
+                    f"Orderbook: {order_book}"
+                 )
+                raise PricingError from e
+            logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side "
+                        f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}")
+            used_rate = rate_from_l2
+        else:
+            logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price")
+            ticker = self.fetch_ticker(pair)
+            ticker_rate = ticker[bid_strategy['price_side']]
+            if ticker['last'] and ticker_rate > ticker['last']:
+                balance = bid_strategy['ask_last_balance']
+                ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate)
+            used_rate = ticker_rate
+
+        self._buy_rate_cache[pair] = used_rate
+
+        return used_rate
+
+    def get_sell_rate(self, pair: str, refresh: bool) -> float:
+        """
+        Get sell rate - either using ticker bid or first bid based on orderbook
+        or remain static in any other case since it's not updating.
+        :param pair: Pair to get rate for
+        :param refresh: allow cached data
+        :return: Bid rate
+        :raises PricingError if price could not be determined.
+        """
+        if not refresh:
+            rate = self._sell_rate_cache.get(pair)
+            # Check if cache has been invalidated
+            if rate:
+                logger.debug(f"Using cached sell rate for {pair}.")
+                return rate
+
+        ask_strategy = self._config.get('ask_strategy', {})
+        if ask_strategy.get('use_order_book', False):
+            # This code is only used for notifications, selling uses the generator directly
+            logger.info(
+                f"Getting price from order book {ask_strategy['price_side'].capitalize()} side."
+            )
+            try:
+                rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s"))
+            except (IndexError, KeyError) as e:
+                logger.warning("Sell Price at location from orderbook could not be determined.")
+                raise PricingError from e
+        else:
+            ticker = self.fetch_ticker(pair)
+            ticker_rate = ticker[ask_strategy['price_side']]
+            if ticker['last'] and ticker_rate < ticker['last']:
+                balance = ask_strategy.get('bid_last_balance', 0.0)
+                ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last'])
+            rate = ticker_rate
+
+        if rate is None:
+            raise PricingError(f"Sell-Rate for {pair} was empty.")
+        self._sell_rate_cache[pair] = rate
+        return rate
+
+    # Fee handling
+
+    @retrier
+    def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List:
+        """
+        Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id.
+        The "since" argument passed in is coming from the database and is in UTC,
+        as timezone-native datetime object.
+        From the python documentation:
+            > Naive datetime instances are assumed to represent local time
+        Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the
+        transformation from local timezone to UTC.
+        This works for timezones UTC+ since then the result will contain trades from a few hours
+        instead of from the last 5 seconds, however fails for UTC- timezones,
+        since we're then asking for trades with a "since" argument in the future.
+
+        :param order_id order_id: Order-id as given when creating the order
+        :param pair: Pair the order is for
+        :param since: datetime object of the order creation time. Assumes object is in UTC.
+        """
+        if self._config['dry_run']:
+            return []
+        if not self.exchange_has('fetchMyTrades'):
+            return []
+        try:
+            # Allow 5s offset to catch slight time offsets (discovered in #1185)
+            # since needs to be int in milliseconds
+            my_trades = self._api.fetch_my_trades(
+                pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000))
+            matched_trades = [trade for trade in my_trades if trade['order'] == order_id]
+
+            self._log_exchange_response('get_trades_for_order', matched_trades)
+            return matched_trades
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(
+                f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e
+        except ccxt.BaseError as e:
+            raise OperationalException(e) from e
+
+    def get_order_id_conditional(self, order: Dict[str, Any]) -> str:
+        return order['id']
+
+    @retrier
+    def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1,
+                price: float = 1, taker_or_maker: str = 'maker') -> float:
+        try:
+            if self._config['dry_run'] and self._config.get('fee', None) is not None:
+                return self._config['fee']
+            # validate that markets are loaded before trying to get fee
+            if self._api.markets is None or len(self._api.markets) == 0:
+                self._api.load_markets()
+
+            return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount,
+                                           price=price, takerOrMaker=taker_or_maker)['rate']
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(
+                f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') from e
+        except ccxt.BaseError as e:
+            raise OperationalException(e) from e
+
+    @staticmethod
+    def order_has_fee(order: Dict) -> bool:
+        """
+        Verifies if the passed in order dict has the needed keys to extract fees,
+        and that these keys (currency, cost) are not empty.
+        :param order: Order or trade (one trade) dict
+        :return: True if the fee substructure contains currency and cost, false otherwise
+        """
+        if not isinstance(order, dict):
+            return False
+        return ('fee' in order and order['fee'] is not None
+                and (order['fee'].keys() >= {'currency', 'cost'})
+                and order['fee']['currency'] is not None
+                and order['fee']['cost'] is not None
+                )
+
+    def calculate_fee_rate(self, order: Dict) -> Optional[float]:
+        """
+        Calculate fee rate if it's not given by the exchange.
+        :param order: Order or trade (one trade) dict
+        """
+        if order['fee'].get('rate') is not None:
+            return order['fee'].get('rate')
+        fee_curr = order['fee']['currency']
+        # Calculate fee based on order details
+        if fee_curr in self.get_pair_base_currency(order['symbol']):
+            # Base currency - divide by amount
+            return round(
+                order['fee']['cost'] / safe_value_fallback2(order, order, 'filled', 'amount'), 8)
+        elif fee_curr in self.get_pair_quote_currency(order['symbol']):
+            # Quote currency - divide by cost
+            return round(order['fee']['cost'] / order['cost'], 8) if order['cost'] else None
+        else:
+            # If Fee currency is a different currency
+            if not order['cost']:
+                # If cost is None or 0.0 -> falsy, return None
+                return None
+            try:
+                comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency'])
+                tick = self.fetch_ticker(comb)
+
+                fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask')
+                return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8)
+            except ExchangeError:
+                return None
+
+    def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]:
+        """
+        Extract tuple of cost, currency, rate.
+        Requires order_has_fee to run first!
+        :param order: Order or trade (one trade) dict
+        :return: Tuple with cost, currency, rate of the given fee dict
+        """
+        return (order['fee']['cost'],
+                order['fee']['currency'],
+                self.calculate_fee_rate(order))
+
+    # Historic data
+
     def get_historic_ohlcv(self, pair: str, timeframe: str,
                            since_ms: int) -> List:
         """
@@ -862,10 +1351,11 @@ class Exchange:
                 "Fetching pair %s, interval %s, since %s %s...",
                 pair, timeframe, since_ms, s
             )
-
+            params = self._ft_has.get('ohlcv_params', {})
             data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe,
                                                      since=since_ms,
-                                                     limit=self.ohlcv_candle_limit(timeframe))
+                                                     limit=self.ohlcv_candle_limit(timeframe),
+                                                     params=params)
 
             # Some exchanges sort OHLCV in ASC order and others in DESC.
             # Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last)
@@ -894,6 +1384,8 @@ class Exchange:
             raise OperationalException(f'Could not fetch historical candle (OHLCV) data '
                                        f'for pair {pair}. Message: {e}') from e
 
+    # Fetch historic trades
+
     @retrier_async
     async def _async_fetch_trades(self, pair: str,
                                   since: Optional[int] = None,
@@ -1052,268 +1544,6 @@ class Exchange:
             self._async_get_trade_history(pair=pair, since=since,
                                           until=until, from_id=from_id))
 
-    def check_order_canceled_empty(self, order: Dict) -> bool:
-        """
-        Verify if an order has been cancelled without being partially filled
-        :param order: Order dict as returned from fetch_order()
-        :return: True if order has been cancelled without being filled, False otherwise.
-        """
-        return (order.get('status') in ('closed', 'canceled', 'cancelled')
-                and order.get('filled') == 0.0)
-
-    @retrier
-    def cancel_order(self, order_id: str, pair: str) -> Dict:
-        if self._config['dry_run']:
-            order = self._dry_run_open_orders.get(order_id)
-            if order:
-                order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']})
-                return order
-            else:
-                return {}
-
-        try:
-            return self._api.cancel_order(order_id, pair)
-        except ccxt.InvalidOrder as e:
-            raise InvalidOrderException(
-                f'Could not cancel order. Message: {e}') from e
-        except ccxt.DDoSProtection as e:
-            raise DDosProtection(e) from e
-        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
-            raise TemporaryError(
-                f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e
-        except ccxt.BaseError as e:
-            raise OperationalException(e) from e
-
-    # Assign method to cancel_stoploss_order to allow easy overriding in other classes
-    cancel_stoploss_order = cancel_order
-
-    def is_cancel_order_result_suitable(self, corder) -> bool:
-        if not isinstance(corder, dict):
-            return False
-
-        required = ('fee', 'status', 'amount')
-        return all(k in corder for k in required)
-
-    def cancel_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict:
-        """
-        Cancel order returning a result.
-        Creates a fake result if cancel order returns a non-usable result
-        and fetch_order does not work (certain exchanges don't return cancelled orders)
-        :param order_id: Orderid to cancel
-        :param pair: Pair corresponding to order_id
-        :param amount: Amount to use for fake response
-        :return: Result from either cancel_order if usable, or fetch_order
-        """
-        try:
-            corder = self.cancel_order(order_id, pair)
-            if self.is_cancel_order_result_suitable(corder):
-                return corder
-        except InvalidOrderException:
-            logger.warning(f"Could not cancel order {order_id} for {pair}.")
-        try:
-            order = self.fetch_order(order_id, pair)
-        except InvalidOrderException:
-            logger.warning(f"Could not fetch cancelled order {order_id}.")
-            order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
-
-        return order
-
-    @retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
-    def fetch_order(self, order_id: str, pair: str) -> Dict:
-        if self._config['dry_run']:
-            try:
-                order = self._dry_run_open_orders[order_id]
-                return order
-            except KeyError as e:
-                # Gracefully handle errors with dry-run orders.
-                raise InvalidOrderException(
-                    f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e
-        try:
-            return self._api.fetch_order(order_id, pair)
-        except ccxt.OrderNotFound as e:
-            raise RetryableOrderError(
-                f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e
-        except ccxt.InvalidOrder as e:
-            raise InvalidOrderException(
-                f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e
-        except ccxt.DDoSProtection as e:
-            raise DDosProtection(e) from e
-        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
-            raise TemporaryError(
-                f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e
-        except ccxt.BaseError as e:
-            raise OperationalException(e) from e
-
-    # Assign method to fetch_stoploss_order to allow easy overriding in other classes
-    fetch_stoploss_order = fetch_order
-
-    def fetch_order_or_stoploss_order(self, order_id: str, pair: str,
-                                      stoploss_order: bool = False) -> Dict:
-        """
-        Simple wrapper calling either fetch_order or fetch_stoploss_order depending on
-        the stoploss_order parameter
-        :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order.
-        """
-        if stoploss_order:
-            return self.fetch_stoploss_order(order_id, pair)
-        return self.fetch_order(order_id, pair)
-
-    @staticmethod
-    def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]],
-                               range_required: bool = True):
-        """
-        Get next greater value in the list.
-        Used by fetch_l2_order_book if the api only supports a limited range
-        """
-        if not limit_range:
-            return limit
-
-        result = min([x for x in limit_range if limit <= x] + [max(limit_range)])
-        if not range_required and limit > result:
-            # Range is not required - we can use None as parameter.
-            return None
-        return result
-
-    @retrier
-    def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
-        """
-        Get L2 order book from exchange.
-        Can be limited to a certain amount (if supported).
-        Returns a dict in the format
-        {'asks': [price, volume], 'bids': [price, volume]}
-        """
-        limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'],
-                                             self._ft_has['l2_limit_range_required'])
-        try:
-
-            return self._api.fetch_l2_order_book(pair, limit1)
-        except ccxt.NotSupported as e:
-            raise OperationalException(
-                f'Exchange {self._api.name} does not support fetching order book.'
-                f'Message: {e}') from e
-        except ccxt.DDoSProtection as e:
-            raise DDosProtection(e) from e
-        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
-            raise TemporaryError(
-                f'Could not get order book due to {e.__class__.__name__}. Message: {e}') from e
-        except ccxt.BaseError as e:
-            raise OperationalException(e) from e
-
-    @retrier
-    def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List:
-        """
-        Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id.
-        The "since" argument passed in is coming from the database and is in UTC,
-        as timezone-native datetime object.
-        From the python documentation:
-            > Naive datetime instances are assumed to represent local time
-        Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the
-        transformation from local timezone to UTC.
-        This works for timezones UTC+ since then the result will contain trades from a few hours
-        instead of from the last 5 seconds, however fails for UTC- timezones,
-        since we're then asking for trades with a "since" argument in the future.
-
-        :param order_id order_id: Order-id as given when creating the order
-        :param pair: Pair the order is for
-        :param since: datetime object of the order creation time. Assumes object is in UTC.
-        """
-        if self._config['dry_run']:
-            return []
-        if not self.exchange_has('fetchMyTrades'):
-            return []
-        try:
-            # Allow 5s offset to catch slight time offsets (discovered in #1185)
-            # since needs to be int in milliseconds
-            my_trades = self._api.fetch_my_trades(
-                pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000))
-            matched_trades = [trade for trade in my_trades if trade['order'] == order_id]
-
-            return matched_trades
-        except ccxt.DDoSProtection as e:
-            raise DDosProtection(e) from e
-        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
-            raise TemporaryError(
-                f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e
-        except ccxt.BaseError as e:
-            raise OperationalException(e) from e
-
-    @retrier
-    def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1,
-                price: float = 1, taker_or_maker: str = 'maker') -> float:
-        try:
-            if self._config['dry_run'] and self._config.get('fee', None) is not None:
-                return self._config['fee']
-            # validate that markets are loaded before trying to get fee
-            if self._api.markets is None or len(self._api.markets) == 0:
-                self._api.load_markets()
-
-            return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount,
-                                           price=price, takerOrMaker=taker_or_maker)['rate']
-        except ccxt.DDoSProtection as e:
-            raise DDosProtection(e) from e
-        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
-            raise TemporaryError(
-                f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') from e
-        except ccxt.BaseError as e:
-            raise OperationalException(e) from e
-
-    @staticmethod
-    def order_has_fee(order: Dict) -> bool:
-        """
-        Verifies if the passed in order dict has the needed keys to extract fees,
-        and that these keys (currency, cost) are not empty.
-        :param order: Order or trade (one trade) dict
-        :return: True if the fee substructure contains currency and cost, false otherwise
-        """
-        if not isinstance(order, dict):
-            return False
-        return ('fee' in order and order['fee'] is not None
-                and (order['fee'].keys() >= {'currency', 'cost'})
-                and order['fee']['currency'] is not None
-                and order['fee']['cost'] is not None
-                )
-
-    def calculate_fee_rate(self, order: Dict) -> Optional[float]:
-        """
-        Calculate fee rate if it's not given by the exchange.
-        :param order: Order or trade (one trade) dict
-        """
-        if order['fee'].get('rate') is not None:
-            return order['fee'].get('rate')
-        fee_curr = order['fee']['currency']
-        # Calculate fee based on order details
-        if fee_curr in self.get_pair_base_currency(order['symbol']):
-            # Base currency - divide by amount
-            return round(
-                order['fee']['cost'] / safe_value_fallback2(order, order, 'filled', 'amount'), 8)
-        elif fee_curr in self.get_pair_quote_currency(order['symbol']):
-            # Quote currency - divide by cost
-            return round(order['fee']['cost'] / order['cost'], 8) if order['cost'] else None
-        else:
-            # If Fee currency is a different currency
-            if not order['cost']:
-                # If cost is None or 0.0 -> falsy, return None
-                return None
-            try:
-                comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency'])
-                tick = self.fetch_ticker(comb)
-
-                fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask')
-                return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8)
-            except ExchangeError:
-                return None
-
-    def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]:
-        """
-        Extract tuple of cost, currency, rate.
-        Requires order_has_fee to run first!
-        :param order: Order or trade (one trade) dict
-        :return: Tuple with cost, currency, rate of the given fee dict
-        """
-        return (order['fee']['cost'],
-                order['fee']['currency'],
-                self.calculate_fee_rate(order))
-
 
 def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = None) -> bool:
     return exchange_name in ccxt_exchanges(ccxt_module)

freqtrade/exchange/ftx.py

@@ -8,6 +8,7 @@ from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, Invali
                                   OperationalException, TemporaryError)
 from freqtrade.exchange import Exchange
 from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT, retrier
+from freqtrade.misc import safe_value_fallback2
 
 
 logger = logging.getLogger(__name__)
@@ -68,6 +69,7 @@ class Ftx(Exchange):
 
             order = self._api.create_order(symbol=pair, type=ordertype, side='sell',
                                            amount=amount, params=params)
+            self._log_exchange_response('create_stoploss_order', order)
             logger.info('stoploss order added for %s. '
                         'stop price: %s.', pair, stop_price)
             return order
@@ -92,18 +94,26 @@ class Ftx(Exchange):
     @retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
     def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict:
         if self._config['dry_run']:
-            try:
-                order = self._dry_run_open_orders[order_id]
-                return order
-            except KeyError as e:
-                # Gracefully handle errors with dry-run orders.
-                raise InvalidOrderException(
-                    f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e
+            return self.fetch_dry_run_order(order_id)
+
         try:
             orders = self._api.fetch_orders(pair, None, params={'type': 'stop'})
 
             order = [order for order in orders if order['id'] == order_id]
+            self._log_exchange_response('fetch_stoploss_order', order)
             if len(order) == 1:
+                if order[0].get('status') == 'closed':
+                    # Trigger order was triggered ...
+                    real_order_id = order[0].get('info', {}).get('orderId')
+
+                    order1 = self._api.fetch_order(real_order_id, pair)
+                    self._log_exchange_response('fetch_stoploss_order1', order1)
+                    # Fake type to stop - as this was really a stop order.
+                    order1['id_stop'] = order1['id']
+                    order1['id'] = order_id
+                    order1['type'] = 'stop'
+                    order1['status_stop'] = 'triggered'
+                    return order1
                 return order[0]
             else:
                 raise InvalidOrderException(f"Could not get stoploss order for id {order_id}")
@@ -124,7 +134,9 @@ class Ftx(Exchange):
         if self._config['dry_run']:
             return {}
         try:
-            return self._api.cancel_order(order_id, pair, params={'type': 'stop'})
+            order = self._api.cancel_order(order_id, pair, params={'type': 'stop'})
+            self._log_exchange_response('cancel_stoploss_order', order)
+            return order
         except ccxt.InvalidOrder as e:
             raise InvalidOrderException(
                 f'Could not cancel order. Message: {e}') from e
@@ -135,3 +147,8 @@ class Ftx(Exchange):
                 f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
+
+    def get_order_id_conditional(self, order: Dict[str, Any]) -> str:
+        if order['type'] == 'stop':
+            return safe_value_fallback2(order, order, 'id_stop', 'id')
+        return order['id']

freqtrade/exchange/hitbtc.py

@@ -0,0 +1,23 @@
+import logging
+from typing import Dict
+
+from freqtrade.exchange import Exchange
+
+
+logger = logging.getLogger(__name__)
+
+
+class Hitbtc(Exchange):
+    """
+    Hitbtc exchange class. Contains adjustments needed for Freqtrade to work
+    with this exchange.
+
+    Please note that this exchange is not included in the list of exchanges
+    officially supported by the Freqtrade development team. So some features
+    may still not work as expected.
+    """
+
+    _ft_has: Dict = {
+        "ohlcv_candle_limit": 1000,
+        "ohlcv_params": {"sort": "DESC"}
+    }

freqtrade/exchange/kraken.py

@@ -49,10 +49,12 @@ class Kraken(Exchange):
             orders = self._api.fetch_open_orders()
             order_list = [(x["symbol"].split("/")[0 if x["side"] == "sell" else 1],
                            x["remaining"] if x["side"] == "sell" else x["remaining"] * x["price"],
-                           # Don't remove the below comment, this can be important for debuggung
+                           # Don't remove the below comment, this can be important for debugging
                            # x["side"], x["amount"],
                            ) for x in orders]
             for bal in balances:
+                if not isinstance(balances[bal], dict):
+                    continue
                 balances[bal]['used'] = sum(order[1] for order in order_list if order[0] == bal)
                 balances[bal]['free'] = balances[bal]['total'] - balances[bal]['used']
 
@@ -101,6 +103,7 @@ class Kraken(Exchange):
 
             order = self._api.create_order(symbol=pair, type=ordertype, side='sell',
                                            amount=amount, price=stop_price, params=params)
+            self._log_exchange_response('create_stoploss_order', order)
             logger.info('stoploss order added for %s. '
                         'stop price: %s.', pair, stop_price)
             return order

freqtrade/freqtradebot.py

@@ -10,13 +10,13 @@ from threading import Lock
 from typing import Any, Dict, List, Optional
 
 import arrow
-from cachetools import TTLCache
 
 from freqtrade import __version__, constants
 from freqtrade.configuration import validate_config_consistency
 from freqtrade.data.converter import order_book_to_dataframe
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.edge import Edge
+from freqtrade.enums import RPCMessageType, SellType, State
 from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError,
                                   InvalidOrderException, PricingError)
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
@@ -26,9 +26,8 @@ from freqtrade.persistence import Order, PairLocks, Trade, cleanup_db, init_db
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
-from freqtrade.rpc import RPCManager, RPCMessageType
-from freqtrade.state import State
-from freqtrade.strategy.interface import IStrategy, SellType
+from freqtrade.rpc import RPCManager
+from freqtrade.strategy.interface import IStrategy, SellCheckTuple
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
 from freqtrade.wallets import Wallets
 
@@ -48,6 +47,7 @@ class FreqtradeBot(LoggingMixin):
         :param config: configuration dict, you can use Configuration.get_config()
         to get the config dict.
         """
+        self.active_pair_whitelist: List[str] = []
 
         logger.info('Starting freqtrade %s', __version__)
 
@@ -57,12 +57,6 @@ class FreqtradeBot(LoggingMixin):
         # Init objects
         self.config = config
 
-        # Cache values for 1800 to avoid frequent polling of the exchange for prices
-        # Caching only applies to RPC methods, so prices for open trades are still
-        # refreshed once every iteration.
-        self._sell_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800)
-        self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800)
-
         self.strategy: IStrategy = StrategyResolver.load_strategy(self.config)
 
         # Check config consistency here since strategies can set certain options
@@ -76,12 +70,19 @@ class FreqtradeBot(LoggingMixin):
 
         PairLocks.timeframe = self.config['timeframe']
 
+        self.protections = ProtectionManager(self.config, self.strategy.protections)
+
+        # RPC runs in separate threads, can start handling external commands just after
+        # initialization, even before Freqtradebot has a chance to start its throttling,
+        # so anything in the Freqtradebot instance should be ready (initialized), including
+        # the initial state of the bot.
+        # Keep this at the end of this initialization method.
+        self.rpc: RPCManager = RPCManager(self)
+
         self.pairlists = PairListManager(self.exchange, self.config)
 
         self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists)
 
-        self.protections = ProtectionManager(self.config)
-
         # Attach Dataprovider to Strategy baseclass
         IStrategy.dp = self.dataprovider
         # Attach Wallets to Strategy baseclass
@@ -97,13 +98,7 @@ class FreqtradeBot(LoggingMixin):
         initial_state = self.config.get('initial_state')
         self.state = State[initial_state.upper()] if initial_state else State.STOPPED
 
-        # RPC runs in separate threads, can start handling external commands just after
-        # initialization, even before Freqtradebot has a chance to start its throttling,
-        # so anything in the Freqtradebot instance should be ready (initialized), including
-        # the initial state of the bot.
-        # Keep this at the end of this initialization method.
-        self.rpc: RPCManager = RPCManager(self)
-        # Protect sell-logic from forcesell and viceversa
+        # Protect sell-logic from forcesell and vice versa
         self._sell_lock = Lock()
         LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe))
 
@@ -187,7 +182,7 @@ class FreqtradeBot(LoggingMixin):
         if self.get_free_open_trades():
             self.enter_positions()
 
-        Trade.query.session.flush()
+        Trade.commit()
 
     def process_stopped(self) -> None:
         """
@@ -267,7 +262,7 @@ class FreqtradeBot(LoggingMixin):
     def update_closed_trades_without_assigned_fees(self):
         """
         Update closed trades without close fees assigned.
-        Only acts when Orders are in the database, otherwise the last orderid is unknown.
+        Only acts when Orders are in the database, otherwise the last order-id is unknown.
         """
         if self.config['dry_run']:
             # Updating open orders in dry-run does not make sense and will fail.
@@ -342,7 +337,7 @@ class FreqtradeBot(LoggingMixin):
                         # Assume this as the open order
                         trade.open_order_id = order.order_id
                 if fo:
-                    logger.info(f"Found {order} for trade {trade}.jj")
+                    logger.info(f"Found {order} for trade {trade}.")
                     self.update_trade_state(trade, order.order_id, fo,
                                             stoploss_order=order.ft_order_side == 'stoploss')
 
@@ -394,51 +389,6 @@ class FreqtradeBot(LoggingMixin):
 
         return trades_created
 
-    def get_buy_rate(self, pair: str, refresh: bool) -> float:
-        """
-        Calculates bid target between current ask price and last price
-        :param pair: Pair to get rate for
-        :param refresh: allow cached data
-        :return: float: Price
-        """
-        if not refresh:
-            rate = self._buy_rate_cache.get(pair)
-            # Check if cache has been invalidated
-            if rate:
-                logger.debug(f"Using cached buy rate for {pair}.")
-                return rate
-
-        bid_strategy = self.config.get('bid_strategy', {})
-        if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False):
-
-            order_book_top = bid_strategy.get('order_book_top', 1)
-            order_book = self.exchange.fetch_l2_order_book(pair, order_book_top)
-            logger.debug('order_book %s', order_book)
-            # top 1 = index 0
-            try:
-                rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0]
-            except (IndexError, KeyError) as e:
-                logger.warning(
-                    "Buy Price from orderbook could not be determined."
-                    f"Orderbook: {order_book}"
-                 )
-                raise PricingError from e
-            logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side "
-                        f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}")
-            used_rate = rate_from_l2
-        else:
-            logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price")
-            ticker = self.exchange.fetch_ticker(pair)
-            ticker_rate = ticker[bid_strategy['price_side']]
-            if ticker['last'] and ticker_rate > ticker['last']:
-                balance = bid_strategy['ask_last_balance']
-                ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate)
-            used_rate = ticker_rate
-
-        self._buy_rate_cache[pair] = used_rate
-
-        return used_rate
-
     def create_trade(self, pair: str) -> bool:
         """
         Check the implemented trading strategy for buy signals.
@@ -522,6 +472,7 @@ class FreqtradeBot(LoggingMixin):
         """
         Executes a limit buy for the given pair
         :param pair: pair for which we want to create a LIMIT_BUY
+        :param stake_amount: amount of stake-currency for the pair
         :return: True if a buy order is created, false if it fails.
         """
         time_in_force = self.strategy.order_time_in_force['buy']
@@ -530,7 +481,7 @@ class FreqtradeBot(LoggingMixin):
             buy_limit_requested = price
         else:
             # Calculate price
-            buy_limit_requested = self.get_buy_rate(pair, True)
+            buy_limit_requested = self.exchange.get_buy_rate(pair, True)
 
         if not buy_limit_requested:
             raise PricingError('Could not determine buy price.')
@@ -552,7 +503,7 @@ class FreqtradeBot(LoggingMixin):
 
         if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)(
                 pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested,
-                time_in_force=time_in_force):
+                time_in_force=time_in_force, current_time=datetime.now(timezone.utc)):
             logger.info(f"User requested abortion of buying {pair}")
             return False
         amount = self.exchange.amount_to_precision(pair, amount)
@@ -601,6 +552,7 @@ class FreqtradeBot(LoggingMixin):
             pair=pair,
             stake_amount=stake_amount,
             amount=amount,
+            is_open=True,
             amount_requested=amount_requested,
             fee_open=fee,
             fee_close=fee,
@@ -619,7 +571,7 @@ class FreqtradeBot(LoggingMixin):
             self.update_trade_state(trade, order_id, order)
 
         Trade.query.session.add(trade)
-        Trade.query.session.flush()
+        Trade.commit()
 
         # Updating wallets
         self.wallets.update()
@@ -630,7 +582,7 @@ class FreqtradeBot(LoggingMixin):
 
     def _notify_buy(self, trade: Trade, order_type: str) -> None:
         """
-        Sends rpc notification when a buy occured.
+        Sends rpc notification when a buy occurred.
         """
         msg = {
             'trade_id': trade.id,
@@ -652,9 +604,9 @@ class FreqtradeBot(LoggingMixin):
 
     def _notify_buy_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
         """
-        Sends rpc notification when a buy cancel occured.
+        Sends rpc notification when a buy cancel occurred.
         """
-        current_rate = self.get_buy_rate(trade.pair, False)
+        current_rate = self.exchange.get_buy_rate(trade.pair, False)
 
         msg = {
             'trade_id': trade.id,
@@ -705,6 +657,7 @@ class FreqtradeBot(LoggingMixin):
                 if (self.strategy.order_types.get('stoploss_on_exchange') and
                         self.handle_stoploss_on_exchange(trade)):
                     trades_closed += 1
+                    Trade.commit()
                     continue
                 # Check if we can sell our current pair
                 if trade.open_order_id is None and trade.is_open and self.handle_trade(trade):
@@ -713,62 +666,12 @@ class FreqtradeBot(LoggingMixin):
             except DependencyException as exception:
                 logger.warning('Unable to sell trade %s: %s', trade.pair, exception)
 
-        # Updating wallets if any trade occured
+        # Updating wallets if any trade occurred
         if trades_closed:
             self.wallets.update()
 
         return trades_closed
 
-    def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1,
-                        order_book_min: int = 1):
-        """
-        Helper generator to query orderbook in loop (used for early sell-order placing)
-        """
-        order_book = self.exchange.fetch_l2_order_book(pair, order_book_max)
-        for i in range(order_book_min, order_book_max + 1):
-            yield order_book[side][i - 1][0]
-
-    def get_sell_rate(self, pair: str, refresh: bool) -> float:
-        """
-        Get sell rate - either using ticker bid or first bid based on orderbook
-        The orderbook portion is only used for rpc messaging, which would otherwise fail
-        for BitMex (has no bid/ask in fetch_ticker)
-        or remain static in any other case since it's not updating.
-        :param pair: Pair to get rate for
-        :param refresh: allow cached data
-        :return: Bid rate
-        """
-        if not refresh:
-            rate = self._sell_rate_cache.get(pair)
-            # Check if cache has been invalidated
-            if rate:
-                logger.debug(f"Using cached sell rate for {pair}.")
-                return rate
-
-        ask_strategy = self.config.get('ask_strategy', {})
-        if ask_strategy.get('use_order_book', False):
-            # This code is only used for notifications, selling uses the generator directly
-            logger.info(
-                f"Getting price from order book {ask_strategy['price_side'].capitalize()} side."
-            )
-            try:
-                rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s"))
-            except (IndexError, KeyError) as e:
-                logger.warning("Sell Price at location from orderbook could not be determined.")
-                raise PricingError from e
-        else:
-            ticker = self.exchange.fetch_ticker(pair)
-            ticker_rate = ticker[ask_strategy['price_side']]
-            if ticker['last'] and ticker_rate < ticker['last']:
-                balance = ask_strategy.get('bid_last_balance', 0.0)
-                ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last'])
-            rate = ticker_rate
-
-        if rate is None:
-            raise PricingError(f"Sell-Rate for {pair} was empty.")
-        self._sell_rate_cache[pair] = rate
-        return rate
-
     def handle_trade(self, trade: Trade) -> bool:
         """
         Sells the current pair if the threshold is reached and updates the trade record.
@@ -796,9 +699,9 @@ class FreqtradeBot(LoggingMixin):
             logger.debug(f'Using order book between {order_book_min} and {order_book_max} '
                          f'for selling {trade.pair}...')
 
-            order_book = self._order_book_gen(trade.pair, f"{config_ask_strategy['price_side']}s",
-                                              order_book_min=order_book_min,
-                                              order_book_max=order_book_max)
+            order_book = self.exchange._order_book_gen(
+                trade.pair, f"{config_ask_strategy['price_side']}s",
+                order_book_min=order_book_min, order_book_max=order_book_max)
             for i in range(order_book_min, order_book_max + 1):
                 try:
                     sell_rate = next(order_book)
@@ -811,14 +714,14 @@ class FreqtradeBot(LoggingMixin):
                              f"{sell_rate:0.8f}")
                 # Assign sell-rate to cache - otherwise sell-rate is never updated in the cache,
                 # resulting in outdated RPC messages
-                self._sell_rate_cache[trade.pair] = sell_rate
+                self.exchange._sell_rate_cache[trade.pair] = sell_rate
 
                 if self._check_and_execute_sell(trade, sell_rate, buy, sell):
                     return True
 
         else:
             logger.debug('checking sell')
-            sell_rate = self.get_sell_rate(trade.pair, True)
+            sell_rate = self.exchange.get_sell_rate(trade.pair, True)
             if self._check_and_execute_sell(trade, sell_rate, buy, sell):
                 return True
 
@@ -850,7 +753,8 @@ class FreqtradeBot(LoggingMixin):
             trade.stoploss_order_id = None
             logger.error(f'Unable to place a stoploss order on exchange. {e}')
             logger.warning('Selling the trade forcefully')
-            self.execute_sell(trade, trade.stop_loss, sell_reason=SellType.EMERGENCY_SELL)
+            self.execute_sell(trade, trade.stop_loss, sell_reason=SellCheckTuple(
+                sell_type=SellType.EMERGENCY_SELL))
 
         except ExchangeError:
             trade.stoploss_order_id = None
@@ -913,8 +817,13 @@ class FreqtradeBot(LoggingMixin):
                 logger.warning('Stoploss order was cancelled, but unable to recreate one.')
 
         # Finally we check if stoploss on exchange should be moved up because of trailing.
-        if stoploss_order and (self.config.get('trailing_stop', False)
-                               or self.config.get('use_custom_stoploss', False)):
+        # Triggered Orders are now real orders - so don't replace stoploss anymore
+        if (
+            stoploss_order
+            and stoploss_order.get('status_stop') != 'triggered'
+            and (self.config.get('trailing_stop', False)
+                 or self.config.get('use_custom_stoploss', False))
+        ):
             # if trailing stoploss is enabled we check if stoploss value has changed
             # in which case we cancel stoploss order and put another one with new
             # value immediately
@@ -926,19 +835,20 @@ class FreqtradeBot(LoggingMixin):
         """
         Check to see if stoploss on exchange should be updated
         in case of trailing stoploss on exchange
-        :param Trade: Corresponding Trade
+        :param trade: Corresponding Trade
         :param order: Current on exchange stoploss order
         :return: None
         """
         if self.exchange.stoploss_adjust(trade.stop_loss, order):
-            # we check if the update is neccesary
+            # we check if the update is necessary
             update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60)
             if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat:
                 # cancelling the current stoploss on exchange first
                 logger.info(f"Cancelling current stoploss on exchange for pair {trade.pair} "
                             f"(orderid:{order['id']}) in order to add another one ...")
                 try:
-                    co = self.exchange.cancel_stoploss_order(order['id'], trade.pair)
+                    co = self.exchange.cancel_stoploss_order_with_result(order['id'], trade.pair,
+                                                                         trade.amount)
                     trade.update_order(co)
                 except InvalidOrderException:
                     logger.exception(f"Could not cancel stoploss order {order['id']} "
@@ -961,7 +871,7 @@ class FreqtradeBot(LoggingMixin):
 
         if should_sell.sell_flag:
             logger.info(f'Executing Sell for {trade.pair}. Reason: {should_sell.sell_type}')
-            self.execute_sell(trade, sell_rate, should_sell.sell_type)
+            self.execute_sell(trade, sell_rate, should_sell)
             return True
         return False
 
@@ -972,15 +882,16 @@ class FreqtradeBot(LoggingMixin):
         timeout = self.config.get('unfilledtimeout', {}).get(side)
         ordertime = arrow.get(order['datetime']).datetime
         if timeout is not None:
-            timeout_threshold = arrow.utcnow().shift(minutes=-timeout).datetime
-
+            timeout_unit = self.config.get('unfilledtimeout', {}).get('unit', 'minutes')
+            timeout_kwargs = {timeout_unit: -timeout}
+            timeout_threshold = arrow.utcnow().shift(**timeout_kwargs).datetime
             return (order['status'] == 'open' and order['side'] == side
                     and ordertime < timeout_threshold)
         return False
 
     def check_handle_timedout(self) -> None:
         """
-        Check if any orders are timed out and cancel if neccessary
+        Check if any orders are timed out and cancel if necessary
         :param timeoutvalue: Number of minutes until order is considered timed out
         :return: None
         """
@@ -1032,6 +943,7 @@ class FreqtradeBot(LoggingMixin):
 
             elif order['side'] == 'sell':
                 self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['ALL_CANCELLED'])
+        Trade.commit()
 
     def handle_cancel_buy(self, trade: Trade, order: Dict, reason: str) -> bool:
         """
@@ -1042,6 +954,16 @@ class FreqtradeBot(LoggingMixin):
 
         # Cancelled orders may have the status of 'canceled' or 'closed'
         if order['status'] not in ('cancelled', 'canceled', 'closed'):
+            filled_val = order.get('filled', 0.0) or 0.0
+            filled_stake = filled_val * trade.open_rate
+            minstake = self.exchange.get_min_pair_stake_amount(
+                trade.pair, trade.open_rate, self.strategy.stoploss)
+
+            if filled_val > 0 and filled_stake < minstake:
+                logger.warning(
+                    f"Order {trade.open_order_id} for {trade.pair} not cancelled, "
+                    f"as the filled amount of {filled_val} would result in an unsellable trade.")
+                return False
             corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
                                                             trade.amount)
             # Avoid race condition where the order could not be cancelled coz its already filled.
@@ -1150,16 +1072,16 @@ class FreqtradeBot(LoggingMixin):
             raise DependencyException(
                 f"Not enough amount to sell. Trade-amount: {amount}, Wallet: {wallet_amount}")
 
-    def execute_sell(self, trade: Trade, limit: float, sell_reason: SellType) -> bool:
+    def execute_sell(self, trade: Trade, limit: float, sell_reason: SellCheckTuple) -> bool:
         """
         Executes a limit sell for the given trade and limit
         :param trade: Trade instance
         :param limit: limit rate for the sell order
-        :param sellreason: Reason the sell was triggered
+        :param sell_reason: Reason the sell was triggered
         :return: True if it succeeds (supported) False (not supported)
         """
         sell_type = 'sell'
-        if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS):
+        if sell_reason.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS):
             sell_type = 'stoploss'
 
         # if stoploss is on exchange and we are on dry_run mode,
@@ -1171,15 +1093,17 @@ class FreqtradeBot(LoggingMixin):
         # First cancelling stoploss on exchange ...
         if self.strategy.order_types.get('stoploss_on_exchange') and trade.stoploss_order_id:
             try:
-                self.exchange.cancel_stoploss_order(trade.stoploss_order_id, trade.pair)
+                co = self.exchange.cancel_stoploss_order_with_result(trade.stoploss_order_id,
+                                                                     trade.pair, trade.amount)
+                trade.update_order(co)
             except InvalidOrderException:
                 logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id}")
 
         order_type = self.strategy.order_types[sell_type]
-        if sell_reason == SellType.EMERGENCY_SELL:
+        if sell_reason.sell_type == SellType.EMERGENCY_SELL:
             # Emergency sells (default to market!)
             order_type = self.strategy.order_types.get("emergencysell", "market")
-        if sell_reason == SellType.FORCE_SELL:
+        if sell_reason.sell_type == SellType.FORCE_SELL:
             # Force sells (default to the sell_type defined in the strategy,
             # but we allow this value to be changed)
             order_type = self.strategy.order_types.get("forcesell", order_type)
@@ -1189,8 +1113,8 @@ class FreqtradeBot(LoggingMixin):
 
         if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)(
                 pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit,
-                time_in_force=time_in_force,
-                sell_reason=sell_reason.value):
+                time_in_force=time_in_force, sell_reason=sell_reason.sell_reason,
+                current_time=datetime.now(timezone.utc)):
             logger.info(f"User requested abortion of selling {trade.pair}")
             return False
 
@@ -1213,13 +1137,13 @@ class FreqtradeBot(LoggingMixin):
         trade.open_order_id = order['id']
         trade.sell_order_status = ''
         trade.close_rate_requested = limit
-        trade.sell_reason = sell_reason.value
+        trade.sell_reason = sell_reason.sell_reason
         # In case of market sell orders the order can be closed immediately
         if order.get('status', 'unknown') == 'closed':
             self.update_trade_state(trade, trade.open_order_id, order)
-        Trade.query.session.flush()
+        Trade.commit()
 
-        # Lock pair for one candle to prevent immediate rebuys
+        # Lock pair for one candle to prevent immediate re-buys
         self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
                                 reason='Auto lock')
 
@@ -1229,12 +1153,12 @@ class FreqtradeBot(LoggingMixin):
 
     def _notify_sell(self, trade: Trade, order_type: str, fill: bool = False) -> None:
         """
-        Sends rpc notification when a sell occured.
+        Sends rpc notification when a sell occurred.
         """
         profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
         profit_trade = trade.calc_profit(rate=profit_rate)
         # Use cached rates here - it was updated seconds ago.
-        current_rate = self.get_sell_rate(trade.pair, False) if not fill else None
+        current_rate = self.exchange.get_sell_rate(trade.pair, False) if not fill else None
         profit_ratio = trade.calc_profit_ratio(profit_rate)
         gain = "profit" if profit_ratio > 0 else "loss"
 
@@ -1270,7 +1194,7 @@ class FreqtradeBot(LoggingMixin):
 
     def _notify_sell_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
         """
-        Sends rpc notification when a sell cancel occured.
+        Sends rpc notification when a sell cancel occurred.
         """
         if trade.sell_order_status == reason:
             return
@@ -1279,7 +1203,7 @@ class FreqtradeBot(LoggingMixin):
 
         profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
         profit_trade = trade.calc_profit(rate=profit_rate)
-        current_rate = self.get_sell_rate(trade.pair, False)
+        current_rate = self.exchange.get_sell_rate(trade.pair, False)
         profit_ratio = trade.calc_profit_ratio(profit_rate)
         gain = "profit" if profit_ratio > 0 else "loss"
 
@@ -1323,7 +1247,7 @@ class FreqtradeBot(LoggingMixin):
         Handles closing both buy and sell orders.
         :param trade: Trade object of the trade we're analyzing
         :param order_id: Order-id of the order we're analyzing
-        :param action_order: Already aquired order object
+        :param action_order: Already acquired order object
         :return: True if order has been cancelled without being filled partially, False otherwise
         """
         if not order_id:
@@ -1358,6 +1282,7 @@ class FreqtradeBot(LoggingMixin):
             # Handling of this will happen in check_handle_timeout.
             return True
         trade.update(order)
+        Trade.commit()
 
         # Updating wallets when order is closed
         if not trade.is_open:
@@ -1393,7 +1318,7 @@ class FreqtradeBot(LoggingMixin):
     def get_real_amount(self, trade: Trade, order: Dict) -> float:
         """
         Detect and update trade fee.
-        Calls trade.update_fee() uppon correct detection.
+        Calls trade.update_fee() upon correct detection.
         Returns modified amount if the fee was taken from the destination currency.
         Necessary for exchanges which charge fees in base currency (e.g. binance)
         :return: identical (or new) amount for the trade
@@ -1426,8 +1351,8 @@ class FreqtradeBot(LoggingMixin):
         """
         fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee.
         """
-        trades = self.exchange.get_trades_for_order(order['id'], trade.pair,
-                                                    trade.open_date)
+        trades = self.exchange.get_trades_for_order(self.exchange.get_order_id_conditional(order),
+                                                    trade.pair, trade.open_date)
 
         if len(trades) == 0:
             logger.info("Applying fee on amount for %s failed: myTrade-Dict empty found", trade)

freqtrade/misc.py

@@ -6,7 +6,7 @@ import logging
 import re
 from datetime import datetime
 from pathlib import Path
-from typing import Any
+from typing import Any, Iterator, List
 from typing.io import IO
 
 import rapidjson
@@ -56,6 +56,7 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool =
     """
     Dump JSON data into a file
     :param filename: file to create
+    :param is_zip: if file should be zip
     :param data: JSON Data to save
     :return:
     """
@@ -202,3 +203,14 @@ def render_template_with_fallback(templatefile: str, templatefallbackfile: str,
         return render_template(templatefile, arguments)
     except TemplateNotFound:
         return render_template(templatefallbackfile, arguments)
+
+
+def chunks(lst: List[Any], n: int) -> Iterator[List[Any]]:
+    """
+    Split lst into chunks of the size n.
+    :param lst: list to split into chunks
+    :param n: number of max elements per chunk
+    :return: None
+    """
+    for chunk in range(0, len(lst), n):
+        yield (lst[chunk:chunk + n])

freqtrade/optimize/backtesting.py

@@ -15,8 +15,9 @@ from freqtrade.configuration import TimeRange, remove_credentials, validate_conf
 from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data import history
 from freqtrade.data.btanalysis import trade_list_to_dataframe
-from freqtrade.data.converter import trim_dataframe
+from freqtrade.data.converter import trim_dataframes
 from freqtrade.data.dataprovider import DataProvider
+from freqtrade.enums import SellType
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
 from freqtrade.mixins import LoggingMixin
@@ -26,7 +27,7 @@ from freqtrade.persistence import LocalTrade, PairLocks, Trade
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
-from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType
+from freqtrade.strategy.interface import IStrategy, SellCheckTuple
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
 from freqtrade.wallets import Wallets
 
@@ -63,9 +64,7 @@ class Backtesting:
         self.all_results: Dict[str, Dict] = {}
 
         self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
-
-        dataprovider = DataProvider(self.config, self.exchange)
-        IStrategy.dp = dataprovider
+        self.dataprovider = DataProvider(self.config, None)
 
         if self.config.get('strategy_list', None):
             for strat in list(self.config['strategy_list']):
@@ -96,7 +95,7 @@ class Backtesting:
                 "PrecisionFilter not allowed for backtesting multiple strategies."
             )
 
-        dataprovider.add_pairlisthandler(self.pairlists)
+        self.dataprovider.add_pairlisthandler(self.pairlists)
         self.pairlists.refresh_pairlist()
 
         if len(self.pairlists.whitelist) == 0:
@@ -112,15 +111,11 @@ class Backtesting:
         PairLocks.timeframe = self.config['timeframe']
         PairLocks.use_db = False
         PairLocks.reset_locks()
-        if self.config.get('enable_protections', False):
-            self.protections = ProtectionManager(self.config)
 
         self.wallets = Wallets(self.config, self.exchange, log=False)
 
         # Get maximum required startup period
         self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
-        # Load one (first) strategy
-        self._set_strategy(self.strategylist[0])
 
     def __del__(self):
         LoggingMixin.show_output = True
@@ -132,10 +127,17 @@ class Backtesting:
         Load strategy into backtesting
         """
         self.strategy: IStrategy = strategy
+        strategy.dp = self.dataprovider
         # Set stoploss_on_exchange to false for backtesting,
         # since a "perfect" stoploss-sell is assumed anyway
         # And the regular "stoploss" function would not apply to that case
         self.strategy.order_types['stoploss_on_exchange'] = False
+        if self.config.get('enable_protections', False):
+            conf = self.config
+            if hasattr(strategy, 'protections'):
+                conf = deepcopy(conf)
+                conf['protections'] = strategy.protections
+            self.protections = ProtectionManager(self.config, strategy.protections)
 
     def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]:
         """
@@ -159,7 +161,7 @@ class Backtesting:
 
         logger.info(f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
-                    f'({(max_date - min_date).days} days)..')
+                    f'({(max_date - min_date).days} days).')
 
         # Adjust startts forward if not enough data is available
         timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
@@ -176,6 +178,8 @@ class Backtesting:
         Trade.use_db = False
         PairLocks.reset_locks()
         Trade.reset_trades()
+        self.rejected_trades = 0
+        self.dataprovider.clear_cache()
 
     def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]:
         """
@@ -189,8 +193,9 @@ class Backtesting:
         data: Dict = {}
         # Create dict with data
         for pair, pair_data in processed.items():
-            pair_data.loc[:, 'buy'] = 0  # cleanup from previous run
-            pair_data.loc[:, 'sell'] = 0  # cleanup from previous run
+            if not pair_data.empty:
+                pair_data.loc[:, 'buy'] = 0  # cleanup if buy_signal is exist
+                pair_data.loc[:, 'sell'] = 0  # cleanup if sell_signal is exist
 
             df_analyzed = self.strategy.advise_sell(
                 self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
@@ -214,6 +219,28 @@ class Backtesting:
         """
         # Special handling if high or low hit STOP_LOSS or ROI
         if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS):
+            if trade.stop_loss > sell_row[HIGH_IDX]:
+                # our stoploss was already higher than candle high,
+                # possibly due to a cancelled trade exit.
+                # sell at open price.
+                return sell_row[OPEN_IDX]
+
+            # Special case: trailing triggers within same candle as trade opened. Assume most
+            # pessimistic price movement, which is moving just enough to arm stoploss and
+            # immediately going down to stop price.
+            if (sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0
+                    and self.strategy.trailing_stop_positive):
+                if self.strategy.trailing_only_offset_is_reached:
+                    # Worst case: price reaches stop_positive_offset and dives down.
+                    stop_rate = (sell_row[OPEN_IDX] *
+                                 (1 + abs(self.strategy.trailing_stop_positive_offset) -
+                                 abs(self.strategy.trailing_stop_positive)))
+                else:
+                    # Worst case: price ticks tiny bit above open and dives down.
+                    stop_rate = sell_row[OPEN_IDX] * (1 - abs(self.strategy.trailing_stop_positive))
+                    assert stop_rate < sell_row[HIGH_IDX]
+                return stop_rate
+
             # Set close_rate to stoploss
             return trade.stop_loss
         elif sell.sell_type == (SellType.ROI):
@@ -250,12 +277,13 @@ class Backtesting:
     def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]:
 
         sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX],  # type: ignore
-                                         sell_row[DATE_IDX], sell_row[BUY_IDX], sell_row[SELL_IDX],
+                                         sell_row[DATE_IDX].to_pydatetime(), sell_row[BUY_IDX],
+                                         sell_row[SELL_IDX],
                                          low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX])
 
         if sell.sell_flag:
-            trade.close_date = sell_row[DATE_IDX]
-            trade.sell_reason = sell.sell_type.value
+            trade.close_date = sell_row[DATE_IDX].to_pydatetime()
+            trade.sell_reason = sell.sell_reason
             trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60)
             closerate = self._get_close_rate(sell_row, trade, sell, trade_dur)
 
@@ -265,7 +293,8 @@ class Backtesting:
                     pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount,
                     rate=closerate,
                     time_in_force=time_in_force,
-                    sell_reason=sell.sell_type.value):
+                    sell_reason=sell.sell_reason,
+                    current_time=sell_row[DATE_IDX].to_pydatetime()):
                 return None
 
             trade.close(closerate, show_msg=False)
@@ -285,7 +314,7 @@ class Backtesting:
         # Confirm trade entry:
         if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)(
                 pair=pair, order_type=order_type, amount=stake_amount, rate=row[OPEN_IDX],
-                time_in_force=time_in_force):
+                time_in_force=time_in_force, current_time=row[DATE_IDX].to_pydatetime()):
             return None
 
         if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount):
@@ -293,7 +322,7 @@ class Backtesting:
             trade = LocalTrade(
                 pair=pair,
                 open_rate=row[OPEN_IDX],
-                open_date=row[DATE_IDX],
+                open_date=row[DATE_IDX].to_pydatetime(),
                 stake_amount=stake_amount,
                 amount=round(stake_amount / row[OPEN_IDX], 8),
                 fee_open=self.fee,
@@ -315,7 +344,7 @@ class Backtesting:
                 for trade in open_trades[pair]:
                     sell_row = data[pair][-1]
 
-                    trade.close_date = sell_row[DATE_IDX]
+                    trade.close_date = sell_row[DATE_IDX].to_pydatetime()
                     trade.sell_reason = SellType.FORCE_SELL.value
                     trade.close(sell_row[OPEN_IDX], show_msg=False)
                     LocalTrade.close_bt_trade(trade)
@@ -325,10 +354,18 @@ class Backtesting:
                     trades.append(trade1)
         return trades
 
+    def trade_slot_available(self, max_open_trades: int, open_trade_count: int) -> bool:
+        # Always allow trades when max_open_trades is enabled.
+        if max_open_trades <= 0 or open_trade_count < max_open_trades:
+            return True
+        # Rejected trade
+        self.rejected_trades += 1
+        return False
+
     def backtest(self, processed: Dict,
                  start_date: datetime, end_date: datetime,
                  max_open_trades: int = 0, position_stacking: bool = False,
-                 enable_protections: bool = False) -> DataFrame:
+                 enable_protections: bool = False) -> Dict[str, Any]:
         """
         Implement backtesting functionality
 
@@ -347,6 +384,10 @@ class Backtesting:
         trades: List[LocalTrade] = []
         self.prepare_backtest(enable_protections)
 
+        # Update dataprovider cache
+        for pair, dataframe in processed.items():
+            self.dataprovider._set_cached_df(pair, self.timeframe, dataframe)
+
         # Use dict of lists with data for performance
         # (looping lists is a lot faster than pandas DataFrames)
         data: Dict = self._get_ohlcv_as_lists(processed)
@@ -363,8 +404,9 @@ class Backtesting:
             open_trade_count_start = open_trade_count
 
             for i, pair in enumerate(data):
+                row_index = indexes[pair]
                 try:
-                    row = data[pair][indexes[pair]]
+                    row = data[pair][row_index]
                 except IndexError:
                     # missing Data for one pair at the end.
                     # Warnings for this are shown during data loading
@@ -373,16 +415,22 @@ class Backtesting:
                 # Waits until the time-counter reaches the start of the data for this pair.
                 if row[DATE_IDX] > tmp:
                     continue
-                indexes[pair] += 1
+
+                row_index += 1
+                self.dataprovider._set_dataframe_max_index(row_index)
+                indexes[pair] = row_index
 
                 # without positionstacking, we can only have one open trade per pair.
                 # max_open_trades must be respected
                 # don't open on the last row
-                if ((position_stacking or len(open_trades[pair]) == 0)
-                        and (max_open_trades <= 0 or open_trade_count_start < max_open_trades)
-                        and tmp != end_date
-                        and row[BUY_IDX] == 1 and row[SELL_IDX] != 1
-                        and not PairLocks.is_pair_locked(pair, row[DATE_IDX])):
+                if (
+                    (position_stacking or len(open_trades[pair]) == 0)
+                    and self.trade_slot_available(max_open_trades, open_trade_count_start)
+                    and tmp != end_date
+                    and row[BUY_IDX] == 1
+                    and row[SELL_IDX] != 1
+                    and not PairLocks.is_pair_locked(pair, row[DATE_IDX])
+                ):
                     trade = self._enter_trade(pair, row)
                     if trade:
                         # TODO: hacky workaround to avoid opening > max_open_trades
@@ -397,7 +445,7 @@ class Backtesting:
                 for trade in open_trades[pair]:
                     # also check the buying candle for sell conditions.
                     trade_entry = self._get_sell_trade_entry(trade, row)
-                    # Sell occured
+                    # Sell occurred
                     if trade_entry:
                         # logger.debug(f"{pair} - Backtesting sell {trade}")
                         open_trade_count -= 1
@@ -415,7 +463,14 @@ class Backtesting:
         trades += self.handle_left_open(open_trades, data=data)
         self.wallets.update()
 
-        return trade_list_to_dataframe(trades)
+        results = trade_list_to_dataframe(trades)
+        return {
+            'results': results,
+            'config': self.strategy.config,
+            'locks': PairLocks.get_all_locks(),
+            'rejected_signals': self.rejected_trades,
+            'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']),
+        }
 
     def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange):
         logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
@@ -437,32 +492,32 @@ class Backtesting:
         preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
 
         # Trim startup period from analyzed dataframe
-        for pair, df in preprocessed.items():
-            preprocessed[pair] = trim_dataframe(df, timerange,
-                                                startup_candles=self.required_startup)
-        min_date, max_date = history.get_timerange(preprocessed)
+        preprocessed = trim_dataframes(preprocessed, timerange, self.required_startup)
 
+        if not preprocessed:
+            raise OperationalException(
+                "No data left after adjusting for startup candles.")
+
+        min_date, max_date = history.get_timerange(preprocessed)
         logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
-                    f'({(max_date - min_date).days} days)..')
+                    f'({(max_date - min_date).days} days).')
         # Execute backtest and store results
         results = self.backtest(
             processed=preprocessed,
-            start_date=min_date.datetime,
-            end_date=max_date.datetime,
+            start_date=min_date,
+            end_date=max_date,
             max_open_trades=max_open_trades,
             position_stacking=self.config.get('position_stacking', False),
             enable_protections=self.config.get('enable_protections', False),
         )
         backtest_end_time = datetime.now(timezone.utc)
-        self.all_results[self.strategy.get_strategy_name()] = {
-            'results': results,
-            'config': self.strategy.config,
-            'locks': PairLocks.get_all_locks(),
-            'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']),
+        results.update({
             'backtest_start_time': int(backtest_start_time.timestamp()),
             'backtest_end_time': int(backtest_end_time.timestamp()),
-        }
+        })
+        self.all_results[self.strategy.get_strategy_name()] = results
+
         return min_date, max_date
 
     def start(self) -> None:
@@ -481,7 +536,7 @@ class Backtesting:
             stats = generate_backtest_stats(data, self.all_results,
                                             min_date=min_date, max_date=max_date)
 
-            if self.config.get('export', False):
+            if self.config.get('export', 'none') == 'trades':
                 store_backtest_stats(self.config['exportfilename'], stats)
 
             # Show backtest results

freqtrade/optimize/hyperopt.py

@@ -4,24 +4,24 @@
 This module contains the hyperopt logic
 """
 
-import locale
 import logging
 import random
 import warnings
-from datetime import datetime
+from datetime import datetime, timezone
 from math import ceil
-from operator import itemgetter
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
+import numpy as np
 import progressbar
+import rapidjson
 from colorama import Fore, Style
 from colorama import init as colorama_init
 from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects
 from pandas import DataFrame
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN
-from freqtrade.data.converter import trim_dataframe
+from freqtrade.data.converter import trim_dataframes
 from freqtrade.data.history import get_timerange
 from freqtrade.misc import file_dump_json, plural
 from freqtrade.optimize.backtesting import Backtesting
@@ -30,8 +30,8 @@ from freqtrade.optimize.hyperopt_auto import HyperOptAuto
 from freqtrade.optimize.hyperopt_interface import IHyperOpt  # noqa: F401
 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss  # noqa: F401
 from freqtrade.optimize.hyperopt_tools import HyperoptTools
+from freqtrade.optimize.optimize_reports import generate_strategy_stats
 from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver
-from freqtrade.strategy import IStrategy
 
 
 # Suppress scikit-learn FutureWarnings from skopt
@@ -65,6 +65,13 @@ class Hyperopt:
     custom_hyperopt: IHyperOpt
 
     def __init__(self, config: Dict[str, Any]) -> None:
+        self.buy_space: List[Dimension] = []
+        self.sell_space: List[Dimension] = []
+        self.roi_space: List[Dimension] = []
+        self.stoploss_space: List[Dimension] = []
+        self.trailing_space: List[Dimension] = []
+        self.dimensions: List[Dimension] = []
+
         self.config = config
 
         self.backtesting = Backtesting(self.config)
@@ -73,15 +80,15 @@ class Hyperopt:
             self.custom_hyperopt = HyperOptAuto(self.config)
         else:
             self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config)
+        self.backtesting._set_strategy(self.backtesting.strategylist[0])
         self.custom_hyperopt.strategy = self.backtesting.strategy
 
         self.custom_hyperoptloss = HyperOptLossResolver.load_hyperoptloss(self.config)
         self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function
         time_now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
         strategy = str(self.config['strategy'])
-        self.results_file = (self.config['user_data_dir'] /
-                             'hyperopt_results' /
-                             f'strategy_{strategy}_hyperopt_results_{time_now}.pickle')
+        self.results_file: Path = (self.config['user_data_dir'] / 'hyperopt_results' /
+                                   f'strategy_{strategy}_{time_now}.fthypt')
         self.data_pickle_file = (self.config['user_data_dir'] /
                                  'hyperopt_results' / 'hyperopt_tickerdata.pkl')
         self.total_epochs = config.get('epochs', 0)
@@ -91,9 +98,7 @@ class Hyperopt:
         self.clean_hyperopt()
 
         self.num_epochs_saved = 0
-
-        # Previous evaluations
-        self.epochs: List = []
+        self.current_best_epoch: Optional[Dict[str, Any]] = None
 
         # Populate functions here (hasattr is slow so should not be run during "regular" operations)
         if hasattr(self.custom_hyperopt, 'populate_indicators'):
@@ -114,7 +119,7 @@ class Hyperopt:
             self.max_open_trades = 0
         self.position_stacking = self.config.get('position_stacking', False)
 
-        if self.has_space('sell'):
+        if HyperoptTools.has_space(self.config, 'sell'):
             # Make sure use_sell_signal is enabled
             if 'ask_strategy' not in self.config:
                 self.config['ask_strategy'] = {}
@@ -140,9 +145,7 @@ class Hyperopt:
                 logger.info(f"Removing `{p}`.")
                 p.unlink()
 
-    def _get_params_dict(self, raw_params: List[Any]) -> Dict:
-
-        dimensions: List[Dimension] = self.dimensions
+    def _get_params_dict(self, dimensions: List[Dimension], raw_params: List[Any]) -> Dict:
 
         # Ensure the number of dimensions match
         # the number of parameters in the list.
@@ -153,21 +156,30 @@ class Hyperopt:
         # and the values are taken from the list of parameters.
         return {d.name: v for d, v in zip(dimensions, raw_params)}
 
-    def _save_results(self) -> None:
+    def _save_result(self, epoch: Dict) -> None:
         """
         Save hyperopt results to file
+        Store one line per epoch.
+        While not a valid json object - this allows appending easily.
+        :param epoch: result dictionary for this epoch.
         """
-        num_epochs = len(self.epochs)
-        if num_epochs > self.num_epochs_saved:
-            logger.debug(f"Saving {num_epochs} {plural(num_epochs, 'epoch')}.")
-            dump(self.epochs, self.results_file)
-            self.num_epochs_saved = num_epochs
-            logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
-                         f"saved to '{self.results_file}'.")
-            # Store hyperopt filename
-            latest_filename = Path.joinpath(self.results_file.parent, LAST_BT_RESULT_FN)
-            file_dump_json(latest_filename, {'latest_hyperopt': str(self.results_file.name)},
-                           log=False)
+        def default_parser(x):
+            if isinstance(x, np.integer):
+                return int(x)
+            return str(x)
+
+        with self.results_file.open('a') as f:
+            rapidjson.dump(epoch, f, default=default_parser,
+                           number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN)
+            f.write("\n")
+
+        self.num_epochs_saved += 1
+        logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
+                     f"saved to '{self.results_file}'.")
+        # Store hyperopt filename
+        latest_filename = Path.joinpath(self.results_file.parent, LAST_BT_RESULT_FN)
+        file_dump_json(latest_filename, {'latest_hyperopt': str(self.results_file.name)},
+                       log=False)
 
     def _get_params_details(self, params: Dict) -> Dict:
         """
@@ -175,18 +187,16 @@ class Hyperopt:
         """
         result: Dict = {}
 
-        if self.has_space('buy'):
-            result['buy'] = {p.name: params.get(p.name)
-                             for p in self.hyperopt_space('buy')}
-        if self.has_space('sell'):
-            result['sell'] = {p.name: params.get(p.name)
-                              for p in self.hyperopt_space('sell')}
-        if self.has_space('roi'):
-            result['roi'] = self.custom_hyperopt.generate_roi_table(params)
-        if self.has_space('stoploss'):
-            result['stoploss'] = {p.name: params.get(p.name)
-                                  for p in self.hyperopt_space('stoploss')}
-        if self.has_space('trailing'):
+        if HyperoptTools.has_space(self.config, 'buy'):
+            result['buy'] = {p.name: params.get(p.name) for p in self.buy_space}
+        if HyperoptTools.has_space(self.config, 'sell'):
+            result['sell'] = {p.name: params.get(p.name) for p in self.sell_space}
+        if HyperoptTools.has_space(self.config, 'roi'):
+            result['roi'] = {str(k): v for k, v in
+                             self.custom_hyperopt.generate_roi_table(params).items()}
+        if HyperoptTools.has_space(self.config, 'stoploss'):
+            result['stoploss'] = {p.name: params.get(p.name) for p in self.stoploss_space}
+        if HyperoptTools.has_space(self.config, 'trailing'):
             result['trailing'] = self.custom_hyperopt.generate_trailing_params(params)
 
         return result
@@ -208,71 +218,58 @@ class Hyperopt:
             )
             self.hyperopt_table_header = 2
 
-    def has_space(self, space: str) -> bool:
+    def init_spaces(self):
         """
-        Tell if the space value is contained in the configuration
+        Assign the dimensions in the hyperoptimization space.
         """
-        # The 'trailing' space is not included in the 'default' set of spaces
-        if space == 'trailing':
-            return any(s in self.config['spaces'] for s in [space, 'all'])
-        else:
-            return any(s in self.config['spaces'] for s in [space, 'all', 'default'])
 
-    def hyperopt_space(self, space: Optional[str] = None) -> List[Dimension]:
-        """
-        Return the dimensions in the hyperoptimization space.
-        :param space: Defines hyperspace to return dimensions for.
-        If None, then the self.has_space() will be used to return dimensions
-        for all hyperspaces used.
-        """
-        spaces: List[Dimension] = []
-
-        if space == 'buy' or (space is None and self.has_space('buy')):
+        if HyperoptTools.has_space(self.config, 'buy'):
             logger.debug("Hyperopt has 'buy' space")
-            spaces += self.custom_hyperopt.indicator_space()
+            self.buy_space = self.custom_hyperopt.indicator_space()
 
-        if space == 'sell' or (space is None and self.has_space('sell')):
+        if HyperoptTools.has_space(self.config, 'sell'):
             logger.debug("Hyperopt has 'sell' space")
-            spaces += self.custom_hyperopt.sell_indicator_space()
+            self.sell_space = self.custom_hyperopt.sell_indicator_space()
 
-        if space == 'roi' or (space is None and self.has_space('roi')):
+        if HyperoptTools.has_space(self.config, 'roi'):
             logger.debug("Hyperopt has 'roi' space")
-            spaces += self.custom_hyperopt.roi_space()
+            self.roi_space = self.custom_hyperopt.roi_space()
 
-        if space == 'stoploss' or (space is None and self.has_space('stoploss')):
+        if HyperoptTools.has_space(self.config, 'stoploss'):
             logger.debug("Hyperopt has 'stoploss' space")
-            spaces += self.custom_hyperopt.stoploss_space()
+            self.stoploss_space = self.custom_hyperopt.stoploss_space()
 
-        if space == 'trailing' or (space is None and self.has_space('trailing')):
+        if HyperoptTools.has_space(self.config, 'trailing'):
             logger.debug("Hyperopt has 'trailing' space")
-            spaces += self.custom_hyperopt.trailing_space()
-
-        return spaces
+            self.trailing_space = self.custom_hyperopt.trailing_space()
+        self.dimensions = (self.buy_space + self.sell_space + self.roi_space +
+                           self.stoploss_space + self.trailing_space)
 
     def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict:
         """
         Used Optimize function. Called once per epoch to optimize whatever is configured.
         Keep this function as optimized as possible!
         """
-        params_dict = self._get_params_dict(raw_params)
-        params_details = self._get_params_details(params_dict)
+        backtest_start_time = datetime.now(timezone.utc)
+        params_dict = self._get_params_dict(self.dimensions, raw_params)
 
-        if self.has_space('roi'):
+        # Apply parameters
+        if HyperoptTools.has_space(self.config, 'roi'):
             self.backtesting.strategy.minimal_roi = (  # type: ignore
                 self.custom_hyperopt.generate_roi_table(params_dict))
 
-        if self.has_space('buy'):
+        if HyperoptTools.has_space(self.config, 'buy'):
             self.backtesting.strategy.advise_buy = (  # type: ignore
                 self.custom_hyperopt.buy_strategy_generator(params_dict))
 
-        if self.has_space('sell'):
+        if HyperoptTools.has_space(self.config, 'sell'):
             self.backtesting.strategy.advise_sell = (  # type: ignore
                 self.custom_hyperopt.sell_strategy_generator(params_dict))
 
-        if self.has_space('stoploss'):
+        if HyperoptTools.has_space(self.config, 'stoploss'):
             self.backtesting.strategy.stoploss = params_dict['stoploss']
 
-        if self.has_space('trailing'):
+        if HyperoptTools.has_space(self.config, 'trailing'):
             d = self.custom_hyperopt.generate_trailing_params(params_dict)
             self.backtesting.strategy.trailing_stop = d['trailing_stop']
             self.backtesting.strategy.trailing_stop_positive = d['trailing_stop_positive']
@@ -281,30 +278,42 @@ class Hyperopt:
             self.backtesting.strategy.trailing_only_offset_is_reached = \
                 d['trailing_only_offset_is_reached']
 
-        processed = load(self.data_pickle_file)
-
-        min_date, max_date = get_timerange(processed)
-
-        backtesting_results = self.backtesting.backtest(
+        with self.data_pickle_file.open('rb') as f:
+            processed = load(f, mmap_mode='r')
+        bt_results = self.backtesting.backtest(
             processed=processed,
-            start_date=min_date.datetime,
-            end_date=max_date.datetime,
+            start_date=self.min_date,
+            end_date=self.max_date,
             max_open_trades=self.max_open_trades,
             position_stacking=self.position_stacking,
             enable_protections=self.config.get('enable_protections', False),
-
         )
-        return self._get_results_dict(backtesting_results, min_date, max_date,
-                                      params_dict, params_details,
+        backtest_end_time = datetime.now(timezone.utc)
+        bt_results.update({
+            'backtest_start_time': int(backtest_start_time.timestamp()),
+            'backtest_end_time': int(backtest_end_time.timestamp()),
+        })
+
+        return self._get_results_dict(bt_results, self.min_date, self.max_date,
+                                      params_dict,
                                       processed=processed)
 
     def _get_results_dict(self, backtesting_results, min_date, max_date,
-                          params_dict, params_details, processed: Dict[str, DataFrame]):
-        results_metrics = self._calculate_results_metrics(backtesting_results)
-        results_explanation = self._format_results_explanation_string(results_metrics)
+                          params_dict, processed: Dict[str, DataFrame]
+                          ) -> Dict[str, Any]:
+        params_details = self._get_params_details(params_dict)
 
-        trade_count = results_metrics['trade_count']
-        total_profit = results_metrics['total_profit']
+        strat_stats = generate_strategy_stats(
+            processed, self.backtesting.strategy.get_strategy_name(),
+            backtesting_results, min_date, max_date, market_change=0
+        )
+        results_explanation = HyperoptTools.format_results_explanation_string(
+            strat_stats, self.config['stake_currency'])
+
+        not_optimized = self.backtesting.strategy.get_params_dict()
+
+        trade_count = strat_stats['total_trades']
+        total_profit = strat_stats['profit_total']
 
         # If this evaluation contains too short amount of trades to be
         # interesting -- consider it as 'bad' (assigned max. loss value)
@@ -312,50 +321,20 @@ class Hyperopt:
         # path. We do not want to optimize 'hodl' strategies.
         loss: float = MAX_LOSS
         if trade_count >= self.config['hyperopt_min_trades']:
-            loss = self.calculate_loss(results=backtesting_results, trade_count=trade_count,
-                                       min_date=min_date.datetime, max_date=max_date.datetime,
+            loss = self.calculate_loss(results=backtesting_results['results'],
+                                       trade_count=trade_count,
+                                       min_date=min_date, max_date=max_date,
                                        config=self.config, processed=processed)
         return {
             'loss': loss,
             'params_dict': params_dict,
             'params_details': params_details,
-            'results_metrics': results_metrics,
+            'params_not_optimized': not_optimized,
+            'results_metrics': strat_stats,
             'results_explanation': results_explanation,
             'total_profit': total_profit,
         }
 
-    def _calculate_results_metrics(self, backtesting_results: DataFrame) -> Dict:
-        wins = len(backtesting_results[backtesting_results['profit_ratio'] > 0])
-        draws = len(backtesting_results[backtesting_results['profit_ratio'] == 0])
-        losses = len(backtesting_results[backtesting_results['profit_ratio'] < 0])
-        return {
-            'trade_count': len(backtesting_results.index),
-            'wins': wins,
-            'draws': draws,
-            'losses': losses,
-            'winsdrawslosses': f"{wins:>4} {draws:>4} {losses:>4}",
-            'avg_profit': backtesting_results['profit_ratio'].mean() * 100.0,
-            'median_profit': backtesting_results['profit_ratio'].median() * 100.0,
-            'total_profit': backtesting_results['profit_abs'].sum(),
-            'profit': backtesting_results['profit_ratio'].sum() * 100.0,
-            'duration': backtesting_results['trade_duration'].mean(),
-        }
-
-    def _format_results_explanation_string(self, results_metrics: Dict) -> str:
-        """
-        Return the formatted results explanation in a string
-        """
-        stake_cur = self.config['stake_currency']
-        return (f"{results_metrics['trade_count']:6d} trades. "
-                f"{results_metrics['wins']}/{results_metrics['draws']}"
-                f"/{results_metrics['losses']} Wins/Draws/Losses. "
-                f"Avg profit {results_metrics['avg_profit']: 6.2f}%. "
-                f"Median profit {results_metrics['median_profit']: 6.2f}%. "
-                f"Total profit {results_metrics['total_profit']: 11.8f} {stake_cur} "
-                f"({results_metrics['profit']: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). "
-                f"Avg duration {results_metrics['duration']:5.1f} min."
-                ).encode(locale.getpreferredencoding(), 'replace').decode('utf-8')
-
     def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
         return Optimizer(
             dimensions,
@@ -374,25 +353,31 @@ class Hyperopt:
     def _set_random_state(self, random_state: Optional[int]) -> int:
         return random_state or random.randint(1, 2**16 - 1)
 
+    def prepare_hyperopt_data(self) -> None:
+        data, timerange = self.backtesting.load_bt_data()
+        logger.info("Dataload complete. Calculating indicators")
+
+        preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
+
+        # Trim startup period from analyzed dataframe
+        processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup)
+
+        self.min_date, self.max_date = get_timerange(processed)
+
+        logger.info(f'Hyperopting with data from {self.min_date.strftime(DATETIME_PRINT_FORMAT)} '
+                    f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} '
+                    f'({(self.max_date - self.min_date).days} days)..')
+
+        dump(processed, self.data_pickle_file)
+
     def start(self) -> None:
         self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
         logger.info(f"Using optimizer random state: {self.random_state}")
         self.hyperopt_table_header = -1
-        data, timerange = self.backtesting.load_bt_data()
-        logger.info("Dataload complete. Calculating indicators")
-        preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
+        # Initialize spaces ...
+        self.init_spaces()
 
-        # Trim startup period from analyzed dataframe
-        for pair, df in preprocessed.items():
-            preprocessed[pair] = trim_dataframe(df, timerange,
-                                                startup_candles=self.backtesting.required_startup)
-        min_date, max_date = get_timerange(preprocessed)
-
-        logger.info(f'Hyperopting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
-                    f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
-                    f'({(max_date - min_date).days} days)..')
-
-        dump(preprocessed, self.data_pickle_file)
+        self.prepare_hyperopt_data()
 
         # We don't need exchange instance anymore while running hyperopt
         self.backtesting.exchange.close()
@@ -400,15 +385,12 @@ class Hyperopt:
         self.backtesting.exchange._api_async = None  # type: ignore
         # self.backtesting.exchange = None  # type: ignore
         self.backtesting.pairlists = None  # type: ignore
-        self.backtesting.strategy.dp = None  # type: ignore
-        IStrategy.dp = None  # type: ignore
 
         cpus = cpu_count()
         logger.info(f"Found {cpus} CPU cores. Let's make them scream!")
         config_jobs = self.config.get('hyperopt_jobs', -1)
         logger.info(f'Number of parallel jobs set as: {config_jobs}')
 
-        self.dimensions: List[Dimension] = self.hyperopt_space()
         self.opt = self.get_optimizer(self.dimensions, config_jobs)
 
         if self.print_colorized:
@@ -474,25 +456,21 @@ class Hyperopt:
 
                             if is_best:
                                 self.current_best_loss = val['loss']
-                            self.epochs.append(val)
+                                self.current_best_epoch = val
 
-                            # Save results after each best epoch and every 100 epochs
-                            if is_best or current % 100 == 0:
-                                self._save_results()
+                            self._save_result(val)
 
                             pbar.update(current)
 
         except KeyboardInterrupt:
             print('User interrupted..')
 
-        self._save_results()
         logger.info(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
                     f"saved to '{self.results_file}'.")
 
-        if self.epochs:
-            sorted_epochs = sorted(self.epochs, key=itemgetter('loss'))
-            best_epoch = sorted_epochs[0]
-            HyperoptTools.print_epoch_details(best_epoch, self.total_epochs, self.print_json)
+        if self.current_best_epoch:
+            HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs,
+                                             self.print_json)
         else:
             # This is printed when Ctrl+C is pressed quickly, before first epochs have
             # a chance to be evaluated.

freqtrade/optimize/hyperopt_loss_onlyprofit.py

@@ -9,23 +9,11 @@ from pandas import DataFrame
 from freqtrade.optimize.hyperopt import IHyperOptLoss
 
 
-# This is assumed to be expected avg profit * expected trade count.
-# For example, for 0.35% avg per trade (or 0.0035 as ratio) and 1100 trades,
-# expected max profit = 3.85
-#
-# Note, this is ratio. 3.85 stated above means 385Σ%, 3.0 means 300Σ%.
-#
-# In this implementation it's only used in calculation of the resulting value
-# of the objective function as a normalization coefficient and does not
-# represent any limit for profits as in the Freqtrade legacy default loss function.
-EXPECTED_MAX_PROFIT = 3.0
-
-
 class OnlyProfitHyperOptLoss(IHyperOptLoss):
     """
     Defines the loss function for hyperopt.
 
-    This implementation takes only profit into account.
+    This implementation takes only absolute profit into account, not looking at any other indicator.
     """
 
     @staticmethod
@@ -34,5 +22,5 @@ class OnlyProfitHyperOptLoss(IHyperOptLoss):
         """
         Objective function, returns smaller number for better results.
         """
-        total_profit = results['profit_ratio'].sum()
-        return 1 - total_profit / EXPECTED_MAX_PROFIT
+        total_profit = results['profit_abs'].sum()
+        return -1 * total_profit

freqtrade/optimize/hyperopt_tools.py

@@ -1,19 +1,16 @@
 
 import io
 import logging
-from collections import OrderedDict
 from pathlib import Path
-from pprint import pformat
-from typing import Dict, List
+from typing import Any, Dict, List
 
 import rapidjson
 import tabulate
 from colorama import Fore, Style
-from joblib import load
 from pandas import isna, json_normalize
 
 from freqtrade.exceptions import OperationalException
-from freqtrade.misc import round_dict
+from freqtrade.misc import round_coin_value, round_dict
 
 
 logger = logging.getLogger(__name__)
@@ -21,13 +18,38 @@ logger = logging.getLogger(__name__)
 
 class HyperoptTools():
 
+    @staticmethod
+    def has_space(config: Dict[str, Any], space: str) -> bool:
+        """
+        Tell if the space value is contained in the configuration
+        """
+        # The 'trailing' space is not included in the 'default' set of spaces
+        if space == 'trailing':
+            return any(s in config['spaces'] for s in [space, 'all'])
+        else:
+            return any(s in config['spaces'] for s in [space, 'all', 'default'])
+
+    @staticmethod
+    def _read_results_pickle(results_file: Path) -> List:
+        """
+        Read hyperopt results from pickle file
+        LEGACY method - new files are written as json and cannot be read with this method.
+        """
+        from joblib import load
+
+        logger.info(f"Reading pickled epochs from '{results_file}'")
+        data = load(results_file)
+        return data
+
     @staticmethod
     def _read_results(results_file: Path) -> List:
         """
         Read hyperopt results from file
         """
-        logger.info("Reading epochs from '%s'", results_file)
-        data = load(results_file)
+        import rapidjson
+        logger.info(f"Reading epochs from '{results_file}'")
+        with results_file.open('r') as f:
+            data = [rapidjson.loads(line) for line in f]
         return data
 
     @staticmethod
@@ -37,7 +59,10 @@ class HyperoptTools():
         """
         epochs: List = []
         if results_file.is_file() and results_file.stat().st_size > 0:
-            epochs = HyperoptTools._read_results(results_file)
+            if results_file.suffix == '.pickle':
+                epochs = HyperoptTools._read_results_pickle(results_file)
+            else:
+                epochs = HyperoptTools._read_results(results_file)
             # Detection of some old format, without 'is_best' field saved
             if epochs[0].get('is_best') is None:
                 raise OperationalException(
@@ -47,12 +72,13 @@ class HyperoptTools():
         return epochs
 
     @staticmethod
-    def print_epoch_details(results, total_epochs: int, print_json: bool,
-                            no_header: bool = False, header_str: str = None) -> None:
+    def show_epoch_details(results, total_epochs: int, print_json: bool,
+                           no_header: bool = False, header_str: str = None) -> None:
         """
         Display details of the hyperopt result
         """
         params = results.get('params_details', {})
+        non_optimized = results.get('params_not_optimized', {})
 
         # Default header string
         if header_str is None:
@@ -65,73 +91,107 @@ class HyperoptTools():
         if print_json:
             result_dict: Dict = {}
             for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']:
-                HyperoptTools._params_update_for_json(result_dict, params, s)
+                HyperoptTools._params_update_for_json(result_dict, params, non_optimized, s)
             print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE))
 
         else:
-            HyperoptTools._params_pretty_print(params, 'buy', "Buy hyperspace params:")
-            HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:")
+            HyperoptTools._params_pretty_print(params, 'buy', "Buy hyperspace params:",
+                                               non_optimized)
+            HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:",
+                                               non_optimized)
             HyperoptTools._params_pretty_print(params, 'roi', "ROI table:")
             HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:")
             HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:")
 
     @staticmethod
-    def _params_update_for_json(result_dict, params, space: str) -> None:
-        if space in params:
+    def _params_update_for_json(result_dict, params, non_optimized, space: str) -> None:
+        if (space in params) or (space in non_optimized):
             space_params = HyperoptTools._space_params(params, space)
-            if space in ['buy', 'sell']:
-                result_dict.setdefault('params', {}).update(space_params)
-            elif space == 'roi':
-                # TODO: get rid of OrderedDict when support for python 3.6 will be
-                # dropped (dicts keep the order as the language feature)
+            space_non_optimized = HyperoptTools._space_params(non_optimized, space)
+            all_space_params = space_params
 
+            # Merge non optimized params if there are any
+            if len(space_non_optimized) > 0:
+                all_space_params = {**space_params, **space_non_optimized}
+
+            if space in ['buy', 'sell']:
+                result_dict.setdefault('params', {}).update(all_space_params)
+            elif space == 'roi':
                 # Convert keys in min_roi dict to strings because
                 # rapidjson cannot dump dicts with integer keys...
-                # OrderedDict is used to keep the numeric order of the items
-                # in the dict.
-                result_dict['minimal_roi'] = OrderedDict(
-                    (str(k), v) for k, v in space_params.items()
-                )
+                result_dict['minimal_roi'] = {str(k): v for k, v in all_space_params.items()}
             else:  # 'stoploss', 'trailing'
-                result_dict.update(space_params)
+                result_dict.update(all_space_params)
 
     @staticmethod
-    def _params_pretty_print(params, space: str, header: str) -> None:
-        if space in params:
+    def _params_pretty_print(params, space: str, header: str, non_optimized={}) -> None:
+        if space in params or space in non_optimized:
             space_params = HyperoptTools._space_params(params, space, 5)
-            params_result = f"\n# {header}\n"
+            result = f"\n# {header}\n"
             if space == 'stoploss':
-                params_result += f"stoploss = {space_params.get('stoploss')}"
+                result += f"stoploss = {space_params.get('stoploss')}"
             elif space == 'roi':
-                # TODO: get rid of OrderedDict when support for python 3.6 will be
-                # dropped (dicts keep the order as the language feature)
-                minimal_roi_result = rapidjson.dumps(
-                    OrderedDict(
-                        (str(k), v) for k, v in space_params.items()
-                    ),
-                    default=str, indent=4, number_mode=rapidjson.NM_NATIVE)
-                params_result += f"minimal_roi = {minimal_roi_result}"
+                minimal_roi_result = rapidjson.dumps({
+                        str(k): v for k, v in space_params.items()
+                }, default=str, indent=4, number_mode=rapidjson.NM_NATIVE)
+                result += f"minimal_roi = {minimal_roi_result}"
             elif space == 'trailing':
 
                 for k, v in space_params.items():
-                    params_result += f'{k} = {v}\n'
+                    result += f'{k} = {v}\n'
 
             else:
-                params_result += f"{space}_params = {pformat(space_params, indent=4)}"
-                params_result = params_result.replace("}", "\n}").replace("{", "{\n ")
+                no_params = HyperoptTools._space_params(non_optimized, space, 5)
 
-            params_result = params_result.replace("\n", "\n    ")
-            print(params_result)
+                result += f"{space}_params = {HyperoptTools._pprint(space_params, no_params)}"
+
+            result = result.replace("\n", "\n    ")
+            print(result)
 
     @staticmethod
     def _space_params(params, space: str, r: int = None) -> Dict:
-        d = params[space]
-        # Round floats to `r` digits after the decimal point if requested
-        return round_dict(d, r) if r else d
+        d = params.get(space)
+        if d:
+            # Round floats to `r` digits after the decimal point if requested
+            return round_dict(d, r) if r else d
+        return {}
+
+    @staticmethod
+    def _pprint(params, non_optimized, indent: int = 4):
+        """
+        Pretty-print hyperopt results (based on 2 dicts - with add. comment)
+        """
+        p = params.copy()
+        p.update(non_optimized)
+        result = '{\n'
+
+        for k, param in p.items():
+            result += " " * indent + f'"{k}": '
+            result += f'"{param}",' if isinstance(param, str) else f'{param},'
+            if k in non_optimized:
+                result += "  # value loaded from strategy"
+            result += "\n"
+        result += '}'
+        return result
 
     @staticmethod
     def is_best_loss(results, current_best_loss: float) -> bool:
-        return results['loss'] < current_best_loss
+        return bool(results['loss'] < current_best_loss)
+
+    @staticmethod
+    def format_results_explanation_string(results_metrics: Dict, stake_currency: str) -> str:
+        """
+        Return the formatted results explanation in a string
+        """
+        return (f"{results_metrics['total_trades']:6d} trades. "
+                f"{results_metrics['wins']}/{results_metrics['draws']}"
+                f"/{results_metrics['losses']} Wins/Draws/Losses. "
+                f"Avg profit {results_metrics['profit_mean'] * 100: 6.2f}%. "
+                f"Median profit {results_metrics['profit_median'] * 100: 6.2f}%. "
+                f"Total profit {results_metrics['profit_total_abs']: 11.8f} {stake_currency} "
+                f"({results_metrics['profit_total'] * 100: 7.2f}%). "
+                f"Avg duration {results_metrics['holding_avg']} min."
+                )
 
     @staticmethod
     def _format_explanation_string(results, total_epochs) -> str:
@@ -140,6 +200,47 @@ class HyperoptTools():
                 f"{results['results_explanation']} " +
                 f"Objective: {results['loss']:.5f}")
 
+    @staticmethod
+    def prepare_trials_columns(trials, legacy_mode: bool, has_drawdown: bool) -> str:
+
+        trials['Best'] = ''
+
+        if 'results_metrics.winsdrawslosses' not in trials.columns:
+            # Ensure compatibility with older versions of hyperopt results
+            trials['results_metrics.winsdrawslosses'] = 'N/A'
+
+        if not has_drawdown:
+            # Ensure compatibility with older versions of hyperopt results
+            trials['results_metrics.max_drawdown_abs'] = None
+            trials['results_metrics.max_drawdown'] = None
+
+        if not legacy_mode:
+            # New mode, using backtest result for metrics
+            trials['results_metrics.winsdrawslosses'] = trials.apply(
+                lambda x: f"{x['results_metrics.wins']} {x['results_metrics.draws']:>4} "
+                          f"{x['results_metrics.losses']:>4}", axis=1)
+            trials = trials[['Best', 'current_epoch', 'results_metrics.total_trades',
+                             'results_metrics.winsdrawslosses',
+                             'results_metrics.profit_mean', 'results_metrics.profit_total_abs',
+                             'results_metrics.profit_total', 'results_metrics.holding_avg',
+                             'results_metrics.max_drawdown', 'results_metrics.max_drawdown_abs',
+                             'loss', 'is_initial_point', 'is_best']]
+
+        else:
+            # Legacy mode
+            trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
+                             'results_metrics.winsdrawslosses', 'results_metrics.avg_profit',
+                             'results_metrics.total_profit', 'results_metrics.profit',
+                             'results_metrics.duration', 'results_metrics.max_drawdown',
+                             'results_metrics.max_drawdown_abs', 'loss', 'is_initial_point',
+                             'is_best']]
+
+        trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
+                          'Total profit', 'Profit', 'Avg duration', 'Max Drawdown',
+                          'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best']
+
+        return trials
+
     @staticmethod
     def get_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool,
                          print_colorized: bool, remove_header: int) -> str:
@@ -150,47 +251,56 @@ class HyperoptTools():
             return ''
 
         tabulate.PRESERVE_WHITESPACE = True
-
         trials = json_normalize(results, max_level=1)
-        trials['Best'] = ''
-        if 'results_metrics.winsdrawslosses' not in trials.columns:
-            # Ensure compatibility with older versions of hyperopt results
-            trials['results_metrics.winsdrawslosses'] = 'N/A'
 
-        trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
-                         'results_metrics.winsdrawslosses',
-                         'results_metrics.avg_profit', 'results_metrics.total_profit',
-                         'results_metrics.profit', 'results_metrics.duration',
-                         'loss', 'is_initial_point', 'is_best']]
-        trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
-                          'Total profit', 'Profit', 'Avg duration', 'Objective',
-                          'is_initial_point', 'is_best']
+        legacy_mode = 'results_metrics.total_trades' not in trials
+        has_drawdown = 'results_metrics.max_drawdown_abs' in trials.columns
+
+        trials = HyperoptTools.prepare_trials_columns(trials, legacy_mode, has_drawdown)
+
         trials['is_profit'] = False
         trials.loc[trials['is_initial_point'], 'Best'] = '*     '
         trials.loc[trials['is_best'], 'Best'] = 'Best'
         trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best'
         trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
         trials['Trades'] = trials['Trades'].astype(str)
-
+        perc_multi = 1 if legacy_mode else 100
         trials['Epoch'] = trials['Epoch'].apply(
             lambda x: '{}/{}'.format(str(x).rjust(len(str(total_epochs)), ' '), total_epochs)
         )
         trials['Avg profit'] = trials['Avg profit'].apply(
-            lambda x: '{:,.2f}%'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
+            lambda x: f'{x * perc_multi:,.2f}%'.rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
         )
         trials['Avg duration'] = trials['Avg duration'].apply(
-            lambda x: '{:,.1f} m'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
+            lambda x: f'{x:,.1f} m'.rjust(7, ' ') if isinstance(x, float) else f"{x}"
+                      if not isna(x) else "--".rjust(7, ' ')
         )
         trials['Objective'] = trials['Objective'].apply(
-            lambda x: '{:,.5f}'.format(x).rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ')
+            lambda x: f'{x:,.5f}'.rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ')
         )
 
+        stake_currency = config['stake_currency']
+
+        if has_drawdown:
+            trials['Max Drawdown'] = trials.apply(
+                lambda x: '{} {}'.format(
+                    round_coin_value(x['max_drawdown_abs'], stake_currency),
+                    '({:,.2f}%)'.format(x['Max Drawdown'] * perc_multi).rjust(10, ' ')
+                ).rjust(25 + len(stake_currency))
+                if x['Max Drawdown'] != 0.0 else '--'.rjust(25 + len(stake_currency)),
+                axis=1
+            )
+        else:
+            trials = trials.drop(columns=['Max Drawdown'])
+
+        trials = trials.drop(columns=['max_drawdown_abs'])
+
         trials['Profit'] = trials.apply(
-            lambda x: '{:,.8f} {} {}'.format(
-                x['Total profit'], config['stake_currency'],
-                '({:,.2f}%)'.format(x['Profit']).rjust(10, ' ')
-            ).rjust(25+len(config['stake_currency']))
-            if x['Total profit'] != 0.0 else '--'.rjust(25+len(config['stake_currency'])),
+            lambda x: '{} {}'.format(
+                round_coin_value(x['Total profit'], stake_currency),
+                '({:,.2f}%)'.format(x['Profit'] * perc_multi).rjust(10, ' ')
+            ).rjust(25+len(stake_currency))
+            if x['Total profit'] != 0.0 else '--'.rjust(25+len(stake_currency)),
             axis=1
         )
         trials = trials.drop(columns=['Total profit'])
@@ -251,11 +361,21 @@ class HyperoptTools():
         trials['Best'] = ''
         trials['Stake currency'] = config['stake_currency']
 
-        base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count',
-                        'results_metrics.avg_profit', 'results_metrics.median_profit',
-                        'results_metrics.total_profit',
-                        'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
-                        'loss', 'is_initial_point', 'is_best']
+        if 'results_metrics.total_trades' in trials:
+            base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades',
+                            'results_metrics.profit_mean', 'results_metrics.profit_median',
+                            'results_metrics.profit_total',
+                            'Stake currency',
+                            'results_metrics.profit_total_abs', 'results_metrics.holding_avg',
+                            'loss', 'is_initial_point', 'is_best']
+            perc_multi = 100
+        else:
+            perc_multi = 1
+            base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count',
+                            'results_metrics.avg_profit', 'results_metrics.median_profit',
+                            'results_metrics.total_profit',
+                            'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
+                            'loss', 'is_initial_point', 'is_best']
         param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()]
         trials = trials[base_metrics + param_metrics]
 
@@ -272,21 +392,24 @@ class HyperoptTools():
         trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
         trials['Epoch'] = trials['Epoch'].astype(str)
         trials['Trades'] = trials['Trades'].astype(str)
+        trials['Median profit'] = trials['Median profit'] * perc_multi
 
         trials['Total profit'] = trials['Total profit'].apply(
-            lambda x: '{:,.8f}'.format(x) if x != 0.0 else ""
+            lambda x: f'{x:,.8f}' if x != 0.0 else ""
         )
         trials['Profit'] = trials['Profit'].apply(
-            lambda x: '{:,.2f}'.format(x) if not isna(x) else ""
+            lambda x: f'{x:,.2f}' if not isna(x) else ""
         )
         trials['Avg profit'] = trials['Avg profit'].apply(
-            lambda x: '{:,.2f}%'.format(x) if not isna(x) else ""
-        )
-        trials['Avg duration'] = trials['Avg duration'].apply(
-            lambda x: '{:,.1f} m'.format(x) if not isna(x) else ""
+            lambda x: f'{x * perc_multi:,.2f}%' if not isna(x) else ""
         )
+        if perc_multi == 1:
+            trials['Avg duration'] = trials['Avg duration'].apply(
+                lambda x: f'{x:,.1f} m' if isinstance(
+                    x, float) else f"{x.total_seconds() // 60:,.1f} m" if not isna(x) else ""
+            )
         trials['Objective'] = trials['Objective'].apply(
-            lambda x: '{:,.5f}'.format(x) if x != 100000 else ""
+            lambda x: f'{x:,.5f}' if x != 100000 else ""
         )
 
         trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit'])

freqtrade/optimize/optimize_reports.py

@@ -3,7 +3,6 @@ from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any, Dict, List, Union
 
-from arrow import Arrow
 from numpy import int64
 from pandas import DataFrame
 from tabulate import tabulate
@@ -22,7 +21,7 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N
     Stores backtest results
     :param recordfilename: Path object, which can either be a filename or a directory.
         Filenames will be appended with a timestamp right before the suffix
-        while for diectories, <directory>/backtest-result-<datetime>.json will be used as filename
+        while for directories, <directory>/backtest-result-<datetime>.json will be used as filename
     :param stats: Dataframe containing the backtesting statistics
     """
     if recordfilename.is_dir():
@@ -44,7 +43,7 @@ def _get_line_floatfmt(stake_currency: str) -> List[str]:
     Generate floatformat (goes in line with _generate_result_line())
     """
     return ['s', 'd', '.2f', '.2f', f'.{decimals_per_coin(stake_currency)}f',
-            '.2f', 'd', 'd', 'd', 'd']
+            '.2f', 'd', 's', 's']
 
 
 def _get_line_header(first_column: str, stake_currency: str) -> List[str]:
@@ -53,7 +52,17 @@ def _get_line_header(first_column: str, stake_currency: str) -> List[str]:
     """
     return [first_column, 'Buys', 'Avg Profit %', 'Cum Profit %',
             f'Tot Profit {stake_currency}', 'Tot Profit %', 'Avg Duration',
-            'Wins', 'Draws', 'Losses']
+            'Win  Draw  Loss  Win%']
+
+
+def _generate_wins_draws_losses(wins, draws, losses):
+    if wins > 0 and losses == 0:
+        wl_ratio = '100'
+    elif wins == 0:
+        wl_ratio = '0'
+    else:
+        wl_ratio = f'{100.0 / (wins + draws + losses) * wins:.1f}' if losses > 0 else '100'
+    return f'{wins:>4}  {draws:>4}  {losses:>4}  {wl_ratio:>4}'
 
 
 def _generate_result_line(result: DataFrame, starting_balance: int, first_column: str) -> Dict:
@@ -153,7 +162,7 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
     return tabular_data
 
 
-def generate_strategy_metrics(all_results: Dict) -> List[Dict]:
+def generate_strategy_comparison(all_results: Dict) -> List[Dict]:
     """
     Generate summary per strategy
     :param all_results: Dict of <Strategyname: DataFrame> containing results for all strategies
@@ -165,6 +174,17 @@ def generate_strategy_metrics(all_results: Dict) -> List[Dict]:
         tabular_data.append(_generate_result_line(
             results['results'], results['config']['dry_run_wallet'], strategy)
             )
+        try:
+            max_drawdown_per, _, _, _, _ = calculate_max_drawdown(results['results'],
+                                                                  value_col='profit_ratio')
+            max_drawdown_abs, _, _, _, _ = calculate_max_drawdown(results['results'],
+                                                                  value_col='profit_abs')
+        except ValueError:
+            max_drawdown_per = 0
+            max_drawdown_abs = 0
+        tabular_data[-1]['max_drawdown_per'] = round(max_drawdown_per * 100, 2)
+        tabular_data[-1]['max_drawdown_abs'] = \
+            round_coin_value(max_drawdown_abs, results['config']['stake_currency'], False)
     return tabular_data
 
 
@@ -194,7 +214,47 @@ def generate_edge_table(results: dict) -> str:
                     floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")  # type: ignore
 
 
+def generate_trading_stats(results: DataFrame) -> Dict[str, Any]:
+    """ Generate overall trade statistics """
+    if len(results) == 0:
+        return {
+            'wins': 0,
+            'losses': 0,
+            'draws': 0,
+            'holding_avg': timedelta(),
+            'winner_holding_avg': timedelta(),
+            'loser_holding_avg': timedelta(),
+        }
+
+    winning_trades = results.loc[results['profit_ratio'] > 0]
+    draw_trades = results.loc[results['profit_ratio'] == 0]
+    losing_trades = results.loc[results['profit_ratio'] < 0]
+    zero_duration_trades = len(results.loc[(results['trade_duration'] == 0) &
+                                           (results['sell_reason'] == 'trailing_stop_loss')])
+
+    holding_avg = (timedelta(minutes=round(results['trade_duration'].mean()))
+                   if not results.empty else timedelta())
+    winner_holding_avg = (timedelta(minutes=round(winning_trades['trade_duration'].mean()))
+                          if not winning_trades.empty else timedelta())
+    loser_holding_avg = (timedelta(minutes=round(losing_trades['trade_duration'].mean()))
+                         if not losing_trades.empty else timedelta())
+
+    return {
+        'wins': len(winning_trades),
+        'losses': len(losing_trades),
+        'draws': len(draw_trades),
+        'holding_avg': holding_avg,
+        'holding_avg_s': holding_avg.total_seconds(),
+        'winner_holding_avg': winner_holding_avg,
+        'winner_holding_avg_s': winner_holding_avg.total_seconds(),
+        'loser_holding_avg': loser_holding_avg,
+        'loser_holding_avg_s': loser_holding_avg.total_seconds(),
+        'zero_duration_trades': zero_duration_trades,
+    }
+
+
 def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
+    """ Generate daily statistics """
     if len(results) == 0:
         return {
             'backtest_best_day': 0,
@@ -204,8 +264,6 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
             'winning_days': 0,
             'draw_days': 0,
             'losing_days': 0,
-            'winner_holding_avg': timedelta(),
-            'loser_holding_avg': timedelta(),
         }
     daily_profit_rel = results.resample('1d', on='close_date')['profit_ratio'].sum()
     daily_profit = results.resample('1d', on='close_date')['profit_abs'].sum().round(10)
@@ -217,9 +275,6 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
     draw_days = sum(daily_profit == 0)
     losing_days = sum(daily_profit < 0)
 
-    winning_trades = results.loc[results['profit_ratio'] > 0]
-    losing_trades = results.loc[results['profit_ratio'] < 0]
-
     return {
         'backtest_best_day': best_rel,
         'backtest_worst_day': worst_rel,
@@ -228,16 +283,152 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
         'winning_days': winning_days,
         'draw_days': draw_days,
         'losing_days': losing_days,
-        'winner_holding_avg': (timedelta(minutes=round(winning_trades['trade_duration'].mean()))
-                               if not winning_trades.empty else timedelta()),
-        'loser_holding_avg': (timedelta(minutes=round(losing_trades['trade_duration'].mean()))
-                              if not losing_trades.empty else timedelta()),
     }
 
 
+def generate_strategy_stats(btdata: Dict[str, DataFrame],
+                            strategy: str,
+                            content: Dict[str, Any],
+                            min_date: datetime, max_date: datetime,
+                            market_change: float
+                            ) -> Dict[str, Any]:
+    """
+    :param btdata: Backtest data
+    :param strategy: Strategy name
+    :param content: Backtest result data in the format:
+                    {'results: results, 'config: config}}.
+    :param min_date: Backtest start date
+    :param max_date: Backtest end date
+    :param market_change: float indicating the market change
+    :return: Dictionary containing results per strategy and a strategy summary.
+    """
+    results: Dict[str, DataFrame] = content['results']
+    if not isinstance(results, DataFrame):
+        return {}
+    config = content['config']
+    max_open_trades = min(config['max_open_trades'], len(btdata.keys()))
+    starting_balance = config['dry_run_wallet']
+    stake_currency = config['stake_currency']
+
+    pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
+                                         starting_balance=starting_balance,
+                                         results=results, skip_nan=False)
+    sell_reason_stats = generate_sell_reason_stats(max_open_trades=max_open_trades,
+                                                   results=results)
+    left_open_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
+                                              starting_balance=starting_balance,
+                                              results=results.loc[results['is_open']],
+                                              skip_nan=True)
+    daily_stats = generate_daily_stats(results)
+    trade_stats = generate_trading_stats(results)
+    best_pair = max([pair for pair in pair_results if pair['key'] != 'TOTAL'],
+                    key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
+    worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'],
+                     key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
+    results['open_timestamp'] = results['open_date'].astype(int64) // 1e6
+    results['close_timestamp'] = results['close_date'].astype(int64) // 1e6
+
+    backtest_days = (max_date - min_date).days
+    strat_stats = {
+        'trades': results.to_dict(orient='records'),
+        'locks': [lock.to_json() for lock in content['locks']],
+        'best_pair': best_pair,
+        'worst_pair': worst_pair,
+        'results_per_pair': pair_results,
+        'sell_reason_summary': sell_reason_stats,
+        'left_open_trades': left_open_results,
+        'total_trades': len(results),
+        'total_volume': float(results['stake_amount'].sum()),
+        'avg_stake_amount': results['stake_amount'].mean() if len(results) > 0 else 0,
+        'profit_mean': results['profit_ratio'].mean() if len(results) > 0 else 0,
+        'profit_median': results['profit_ratio'].median() if len(results) > 0 else 0,
+        'profit_total': results['profit_abs'].sum() / starting_balance,
+        'profit_total_abs': results['profit_abs'].sum(),
+        'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT),
+        'backtest_start_ts': int(min_date.timestamp() * 1000),
+        'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT),
+        'backtest_end_ts': int(max_date.timestamp() * 1000),
+        'backtest_days': backtest_days,
+
+        'backtest_run_start_ts': content['backtest_start_time'],
+        'backtest_run_end_ts': content['backtest_end_time'],
+
+        'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0,
+        'market_change': market_change,
+        'pairlist': list(btdata.keys()),
+        'stake_amount': config['stake_amount'],
+        'stake_currency': config['stake_currency'],
+        'stake_currency_decimals': decimals_per_coin(config['stake_currency']),
+        'starting_balance': starting_balance,
+        'dry_run_wallet': starting_balance,
+        'final_balance': content['final_balance'],
+        'rejected_signals': content['rejected_signals'],
+        'max_open_trades': max_open_trades,
+        'max_open_trades_setting': (config['max_open_trades']
+                                    if config['max_open_trades'] != float('inf') else -1),
+        'timeframe': config['timeframe'],
+        'timerange': config.get('timerange', ''),
+        'enable_protections': config.get('enable_protections', False),
+        'strategy_name': strategy,
+        # Parameters relevant for backtesting
+        'stoploss': config['stoploss'],
+        'trailing_stop': config.get('trailing_stop', False),
+        'trailing_stop_positive': config.get('trailing_stop_positive'),
+        'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset', 0.0),
+        'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False),
+        'use_custom_stoploss': config.get('use_custom_stoploss', False),
+        'minimal_roi': config['minimal_roi'],
+        'use_sell_signal': config['ask_strategy']['use_sell_signal'],
+        'sell_profit_only': config['ask_strategy']['sell_profit_only'],
+        'sell_profit_offset': config['ask_strategy']['sell_profit_offset'],
+        'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'],
+        **daily_stats,
+        **trade_stats
+    }
+
+    try:
+        max_drawdown, _, _, _, _ = calculate_max_drawdown(
+            results, value_col='profit_ratio')
+        drawdown_abs, drawdown_start, drawdown_end, high_val, low_val = calculate_max_drawdown(
+            results, value_col='profit_abs')
+        strat_stats.update({
+            'max_drawdown': max_drawdown,
+            'max_drawdown_abs': drawdown_abs,
+            'drawdown_start': drawdown_start.strftime(DATETIME_PRINT_FORMAT),
+            'drawdown_start_ts': drawdown_start.timestamp() * 1000,
+            'drawdown_end': drawdown_end.strftime(DATETIME_PRINT_FORMAT),
+            'drawdown_end_ts': drawdown_end.timestamp() * 1000,
+
+            'max_drawdown_low': low_val,
+            'max_drawdown_high': high_val,
+        })
+
+        csum_min, csum_max = calculate_csum(results, starting_balance)
+        strat_stats.update({
+            'csum_min': csum_min,
+            'csum_max': csum_max
+        })
+
+    except ValueError:
+        strat_stats.update({
+            'max_drawdown': 0.0,
+            'max_drawdown_abs': 0.0,
+            'max_drawdown_low': 0.0,
+            'max_drawdown_high': 0.0,
+            'drawdown_start': datetime(1970, 1, 1, tzinfo=timezone.utc),
+            'drawdown_start_ts': 0,
+            'drawdown_end': datetime(1970, 1, 1, tzinfo=timezone.utc),
+            'drawdown_end_ts': 0,
+            'csum_min': 0,
+            'csum_max': 0
+        })
+
+    return strat_stats
+
+
 def generate_backtest_stats(btdata: Dict[str, DataFrame],
                             all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]],
-                            min_date: Arrow, max_date: Arrow
+                            min_date: datetime, max_date: datetime
                             ) -> Dict[str, Any]:
     """
     :param btdata: Backtest data
@@ -245,132 +436,17 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
                      { Strategy: {'results: results, 'config: config}}.
     :param min_date: Backtest start date
     :param max_date: Backtest end date
-    :return:
-    Dictionary containing results per strategy and a stratgy summary.
+    :return: Dictionary containing results per strategy and a strategy summary.
     """
     result: Dict[str, Any] = {'strategy': {}}
     market_change = calculate_market_change(btdata, 'close')
 
     for strategy, content in all_results.items():
-        results: Dict[str, DataFrame] = content['results']
-        if not isinstance(results, DataFrame):
-            continue
-        config = content['config']
-        max_open_trades = min(config['max_open_trades'], len(btdata.keys()))
-        starting_balance = config['dry_run_wallet']
-        stake_currency = config['stake_currency']
-
-        pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
-                                             starting_balance=starting_balance,
-                                             results=results, skip_nan=False)
-        sell_reason_stats = generate_sell_reason_stats(max_open_trades=max_open_trades,
-                                                       results=results)
-        left_open_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
-                                                  starting_balance=starting_balance,
-                                                  results=results.loc[results['is_open']],
-                                                  skip_nan=True)
-        daily_stats = generate_daily_stats(results)
-        best_pair = max([pair for pair in pair_results if pair['key'] != 'TOTAL'],
-                        key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
-        worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'],
-                         key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
-        results['open_timestamp'] = results['open_date'].astype(int64) // 1e6
-        results['close_timestamp'] = results['close_date'].astype(int64) // 1e6
-
-        backtest_days = (max_date - min_date).days
-        strat_stats = {
-            'trades': results.to_dict(orient='records'),
-            'locks': [lock.to_json() for lock in content['locks']],
-            'best_pair': best_pair,
-            'worst_pair': worst_pair,
-            'results_per_pair': pair_results,
-            'sell_reason_summary': sell_reason_stats,
-            'left_open_trades': left_open_results,
-            'total_trades': len(results),
-            'total_volume': float(results['stake_amount'].sum()),
-            'avg_stake_amount': results['stake_amount'].mean() if len(results) > 0 else 0,
-            'profit_mean': results['profit_ratio'].mean() if len(results) > 0 else 0,
-            'profit_total': results['profit_abs'].sum() / starting_balance,
-            'profit_total_abs': results['profit_abs'].sum(),
-            'backtest_start': min_date.datetime,
-            'backtest_start_ts': min_date.int_timestamp * 1000,
-            'backtest_end': max_date.datetime,
-            'backtest_end_ts': max_date.int_timestamp * 1000,
-            'backtest_days': backtest_days,
-
-            'backtest_run_start_ts': content['backtest_start_time'],
-            'backtest_run_end_ts': content['backtest_end_time'],
-
-            'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0,
-            'market_change': market_change,
-            'pairlist': list(btdata.keys()),
-            'stake_amount': config['stake_amount'],
-            'stake_currency': config['stake_currency'],
-            'stake_currency_decimals': decimals_per_coin(config['stake_currency']),
-            'starting_balance': starting_balance,
-            'dry_run_wallet': starting_balance,
-            'final_balance': content['final_balance'],
-            'max_open_trades': max_open_trades,
-            'max_open_trades_setting': (config['max_open_trades']
-                                        if config['max_open_trades'] != float('inf') else -1),
-            'timeframe': config['timeframe'],
-            'timerange': config.get('timerange', ''),
-            'enable_protections': config.get('enable_protections', False),
-            'strategy_name': strategy,
-            # Parameters relevant for backtesting
-            'stoploss': config['stoploss'],
-            'trailing_stop': config.get('trailing_stop', False),
-            'trailing_stop_positive': config.get('trailing_stop_positive'),
-            'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset', 0.0),
-            'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False),
-            'use_custom_stoploss': config.get('use_custom_stoploss', False),
-            'minimal_roi': config['minimal_roi'],
-            'use_sell_signal': config['ask_strategy']['use_sell_signal'],
-            'sell_profit_only': config['ask_strategy']['sell_profit_only'],
-            'sell_profit_offset': config['ask_strategy']['sell_profit_offset'],
-            'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'],
-            **daily_stats,
-        }
+        strat_stats = generate_strategy_stats(btdata, strategy, content,
+                                              min_date, max_date, market_change=market_change)
         result['strategy'][strategy] = strat_stats
 
-        try:
-            max_drawdown, _, _, _, _ = calculate_max_drawdown(
-                results, value_col='profit_ratio')
-            drawdown_abs, drawdown_start, drawdown_end, high_val, low_val = calculate_max_drawdown(
-                results, value_col='profit_abs')
-            strat_stats.update({
-                'max_drawdown': max_drawdown,
-                'max_drawdown_abs': drawdown_abs,
-                'drawdown_start': drawdown_start,
-                'drawdown_start_ts': drawdown_start.timestamp() * 1000,
-                'drawdown_end': drawdown_end,
-                'drawdown_end_ts': drawdown_end.timestamp() * 1000,
-
-                'max_drawdown_low': low_val,
-                'max_drawdown_high': high_val,
-            })
-
-            csum_min, csum_max = calculate_csum(results, starting_balance)
-            strat_stats.update({
-                'csum_min': csum_min,
-                'csum_max': csum_max
-            })
-
-        except ValueError:
-            strat_stats.update({
-                'max_drawdown': 0.0,
-                'max_drawdown_abs': 0.0,
-                'max_drawdown_low': 0.0,
-                'max_drawdown_high': 0.0,
-                'drawdown_start': datetime(1970, 1, 1, tzinfo=timezone.utc),
-                'drawdown_start_ts': 0,
-                'drawdown_end': datetime(1970, 1, 1, tzinfo=timezone.utc),
-                'drawdown_end_ts': 0,
-                'csum_min': 0,
-                'csum_max': 0
-            })
-
-    strategy_results = generate_strategy_metrics(all_results=all_results)
+    strategy_results = generate_strategy_comparison(all_results=all_results)
 
     result['strategy_comparison'] = strategy_results
 
@@ -393,7 +469,8 @@ def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: st
     floatfmt = _get_line_floatfmt(stake_currency)
     output = [[
         t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'],
-        t['profit_total_pct'], t['duration_avg'], t['wins'], t['draws'], t['losses']
+        t['profit_total_pct'], t['duration_avg'],
+        _generate_wins_draws_losses(t['wins'], t['draws'], t['losses'])
     ] for t in pair_results]
     # Ignore type as floatfmt does allow tuples but mypy does not know that
     return tabulate(output, headers=headers,
@@ -410,9 +487,7 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
     headers = [
         'Sell Reason',
         'Sells',
-        'Wins',
-        'Draws',
-        'Losses',
+        'Win  Draws  Loss  Win%',
         'Avg Profit %',
         'Cum Profit %',
         f'Tot Profit {stake_currency}',
@@ -420,7 +495,8 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
     ]
 
     output = [[
-        t['sell_reason'], t['trades'], t['wins'], t['draws'], t['losses'],
+        t['sell_reason'], t['trades'],
+        _generate_wins_draws_losses(t['wins'], t['draws'], t['losses']),
         t['profit_mean_pct'], t['profit_sum_pct'],
         round_coin_value(t['profit_total_abs'], stake_currency, False),
         t['profit_total_pct'],
@@ -431,18 +507,28 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
 def text_table_strategy(strategy_results, stake_currency: str) -> str:
     """
     Generate summary table per strategy
+    :param strategy_results: Dict of <Strategyname: DataFrame> containing results for all strategies
     :param stake_currency: stake-currency - used to correctly name headers
-    :param max_open_trades: Maximum allowed open trades used for backtest
-    :param all_results: Dict of <Strategyname: DataFrame> containing results for all strategies
     :return: pretty printed table with tabulate as string
     """
     floatfmt = _get_line_floatfmt(stake_currency)
     headers = _get_line_header('Strategy', stake_currency)
+    # _get_line_header() is also used for per-pair summary. Per-pair drawdown is mostly useless
+    # therefore we slip this column in only for strategy summary here.
+    headers.append('Drawdown')
+
+    # Align drawdown string on the center two space separator.
+    drawdown = [f'{t["max_drawdown_per"]:.2f}' for t in strategy_results]
+    dd_pad_abs = max([len(t['max_drawdown_abs']) for t in strategy_results])
+    dd_pad_per = max([len(dd) for dd in drawdown])
+    drawdown = [f'{t["max_drawdown_abs"]:>{dd_pad_abs}} {stake_currency}  {dd:>{dd_pad_per}}%'
+                for t, dd in zip(strategy_results, drawdown)]
 
     output = [[
         t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'],
-        t['profit_total_pct'], t['duration_avg'], t['wins'], t['draws'], t['losses']
-    ] for t in strategy_results]
+        t['profit_total_pct'], t['duration_avg'],
+        _generate_wins_draws_losses(t['wins'], t['draws'], t['losses']), drawdown]
+        for t, drawdown in zip(strategy_results, drawdown)]
     # Ignore type as floatfmt does allow tuples but mypy does not know that
     return tabulate(output, headers=headers,
                     floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
@@ -452,25 +538,36 @@ def text_table_add_metrics(strat_results: Dict) -> str:
     if len(strat_results['trades']) > 0:
         best_trade = max(strat_results['trades'], key=lambda x: x['profit_ratio'])
         worst_trade = min(strat_results['trades'], key=lambda x: x['profit_ratio'])
+
+        # Newly added fields should be ignored if they are missing in strat_results. hyperopt-show
+        # command stores these results and newer version of freqtrade must be able to handle old
+        # results with missing new fields.
+        zero_duration_trades = '--'
+
+        if 'zero_duration_trades' in strat_results:
+            zero_duration_trades_per = \
+                100.0 / strat_results['total_trades'] * strat_results['zero_duration_trades']
+            zero_duration_trades = f'{zero_duration_trades_per:.2f}% ' \
+                                   f'({strat_results["zero_duration_trades"]})'
+
         metrics = [
-            ('Backtesting from', strat_results['backtest_start'].strftime(DATETIME_PRINT_FORMAT)),
-            ('Backtesting to', strat_results['backtest_end'].strftime(DATETIME_PRINT_FORMAT)),
+            ('Backtesting from', strat_results['backtest_start']),
+            ('Backtesting to', strat_results['backtest_end']),
             ('Max open trades', strat_results['max_open_trades']),
             ('', ''),  # Empty line to improve readability
-            ('Total trades', strat_results['total_trades']),
+            ('Total/Daily Avg Trades',
+                f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"),
             ('Starting balance', round_coin_value(strat_results['starting_balance'],
                                                   strat_results['stake_currency'])),
             ('Final balance', round_coin_value(strat_results['final_balance'],
                                                strat_results['stake_currency'])),
             ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'],
                                                   strat_results['stake_currency'])),
-            ('Total profit %', f"{round(strat_results['profit_total'] * 100, 2)}%"),
-            ('Trades per day', strat_results['trades_per_day']),
+            ('Total profit %', f"{round(strat_results['profit_total'] * 100, 2):}%"),
             ('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'],
                                                    strat_results['stake_currency'])),
             ('Total trade volume', round_coin_value(strat_results['total_volume'],
                                                     strat_results['stake_currency'])),
-
             ('', ''),  # Empty line to improve readability
             ('Best Pair', f"{strat_results['best_pair']['key']} "
                           f"{round(strat_results['best_pair']['profit_sum_pct'], 2)}%"),
@@ -488,6 +585,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
                 f"{strat_results['draw_days']} / {strat_results['losing_days']}"),
             ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"),
             ('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"),
+            ('Zero Duration Trades', zero_duration_trades),
+            ('Rejected Buy signals', strat_results.get('rejected_signals', 'N/A')),
             ('', ''),  # Empty line to improve readability
 
             ('Min balance', round_coin_value(strat_results['csum_min'],
@@ -502,8 +601,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
                                                strat_results['stake_currency'])),
             ('Drawdown low', round_coin_value(strat_results['max_drawdown_low'],
                                               strat_results['stake_currency'])),
-            ('Drawdown Start', strat_results['drawdown_start'].strftime(DATETIME_PRINT_FORMAT)),
-            ('Drawdown End', strat_results['drawdown_end'].strftime(DATETIME_PRINT_FORMAT)),
+            ('Drawdown Start', strat_results['drawdown_start']),
+            ('Drawdown End', strat_results['drawdown_end']),
             ('Market change', f"{round(strat_results['market_change'] * 100, 2)}%"),
         ]
 
@@ -522,37 +621,43 @@ def text_table_add_metrics(strat_results: Dict) -> str:
         return message
 
 
+def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: str):
+    """
+    Print results for one strategy
+    """
+    # Print results
+    print(f"Result for strategy {strategy}")
+    table = text_table_bt_results(results['results_per_pair'], stake_currency=stake_currency)
+    if isinstance(table, str):
+        print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
+    print(table)
+
+    table = text_table_sell_reason(sell_reason_stats=results['sell_reason_summary'],
+                                   stake_currency=stake_currency)
+    if isinstance(table, str) and len(table) > 0:
+        print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
+    print(table)
+
+    table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency)
+    if isinstance(table, str) and len(table) > 0:
+        print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
+    print(table)
+
+    table = text_table_add_metrics(results)
+    if isinstance(table, str) and len(table) > 0:
+        print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '='))
+    print(table)
+
+    if isinstance(table, str) and len(table) > 0:
+        print('=' * len(table.splitlines()[0]))
+    print()
+
+
 def show_backtest_results(config: Dict, backtest_stats: Dict):
     stake_currency = config['stake_currency']
 
     for strategy, results in backtest_stats['strategy'].items():
-
-        # Print results
-        print(f"Result for strategy {strategy}")
-        table = text_table_bt_results(results['results_per_pair'], stake_currency=stake_currency)
-        if isinstance(table, str):
-            print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
-        print(table)
-
-        table = text_table_sell_reason(sell_reason_stats=results['sell_reason_summary'],
-                                       stake_currency=stake_currency)
-        if isinstance(table, str) and len(table) > 0:
-            print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
-        print(table)
-
-        table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency)
-        if isinstance(table, str) and len(table) > 0:
-            print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
-        print(table)
-
-        table = text_table_add_metrics(results)
-        if isinstance(table, str) and len(table) > 0:
-            print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '='))
-        print(table)
-
-        if isinstance(table, str) and len(table) > 0:
-            print('=' * len(table.splitlines()[0]))
-        print()
+        show_backtest_result(strategy, results, stake_currency)
 
     if len(backtest_stats['strategy']) > 1:
         # Print Strategy summary table

freqtrade/persistence/migrations.py

@@ -1,7 +1,7 @@
 import logging
 from typing import List
 
-from sqlalchemy import inspect
+from sqlalchemy import inspect, text
 
 
 logger = logging.getLogger(__name__)
@@ -62,15 +62,17 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
     amount_requested = get_column_def(cols, 'amount_requested', 'amount')
 
     # Schema migration necessary
-    engine.execute(f"alter table trades rename to {table_back_name}")
-    # drop indexes on backup table
-    for index in inspector.get_indexes(table_back_name):
-        engine.execute(f"drop index {index['name']}")
+    with engine.begin() as connection:
+        connection.execute(text(f"alter table trades rename to {table_back_name}"))
+        # drop indexes on backup table
+        for index in inspector.get_indexes(table_back_name):
+            connection.execute(text(f"drop index {index['name']}"))
     # let SQLAlchemy create the schema as required
     decl_base.metadata.create_all(engine)
 
     # Copy data back - following the correct schema
-    engine.execute(f"""insert into trades
+    with engine.begin() as connection:
+        connection.execute(text(f"""insert into trades
             (id, exchange, pair, is_open,
             fee_open, fee_open_cost, fee_open_currency,
             fee_close, fee_close_cost, fee_open_currency, open_rate,
@@ -104,11 +106,12 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
             {strategy} strategy, {timeframe} timeframe,
             {open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs
             from {table_back_name}
-            """)
+            """))
 
 
 def migrate_open_orders_to_trades(engine):
-    engine.execute("""
+    with engine.begin() as connection:
+        connection.execute(text("""
         insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open)
         select id ft_trade_id, pair ft_pair, open_order_id,
             case when close_rate_requested is null then 'buy'
@@ -120,7 +123,30 @@ def migrate_open_orders_to_trades(engine):
             'stoploss' ft_order_side, 1 ft_is_open
         from trades
         where stoploss_order_id is not null
-        """)
+        """))
+
+
+def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, cols: List):
+    # Schema migration necessary
+
+    with engine.begin() as connection:
+        connection.execute(text(f"alter table orders rename to {table_back_name}"))
+        # drop indexes on backup table
+        for index in inspector.get_indexes(table_back_name):
+            connection.execute(text(f"drop index {index['name']}"))
+
+    # let SQLAlchemy create the schema as required
+    decl_base.metadata.create_all(engine)
+    with engine.begin() as connection:
+        connection.execute(text(f"""
+            insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
+            status, symbol, order_type, side, price, amount, filled, average, remaining, cost,
+            order_date, order_filled_date, order_update_date)
+            select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
+            status, symbol, order_type, side, price, amount, filled, null average, remaining, cost,
+            order_date, order_filled_date, order_update_date
+            from {table_back_name}
+            """))
 
 
 def check_migrate(engine, decl_base, previous_tables) -> None:
@@ -145,6 +171,11 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
         logger.info('Moving open orders to Orders table.')
         migrate_open_orders_to_trades(engine)
     else:
-        pass
-        # Empty for now - as there is only one iteration of the orders table so far.
-        # table_back_name = get_backup_name(tabs, 'orders_bak')
+        cols_order = inspector.get_columns('orders')
+
+        if not has_column(cols_order, 'average'):
+            tabs = get_table_names_for_table(inspector, 'orders')
+            # Empty for now - as there is only one iteration of the orders table so far.
+            table_back_name = get_backup_name(tabs, 'orders_bak')
+
+            migrate_orders_table(decl_base, inspector, engine, table_back_name, cols)

freqtrade/persistence/models.py

@@ -9,14 +9,12 @@ from typing import Any, Dict, List, Optional
 from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, String,
                         create_engine, desc, func, inspect)
 from sqlalchemy.exc import NoSuchModuleError
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import Query, relationship
-from sqlalchemy.orm.scoping import scoped_session
-from sqlalchemy.orm.session import sessionmaker
+from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session, sessionmaker
 from sqlalchemy.pool import StaticPool
 from sqlalchemy.sql.schema import UniqueConstraint
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT
+from freqtrade.enums import SellType
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.misc import safe_value_fallback
 from freqtrade.persistence.migrations import check_migrate
@@ -41,16 +39,18 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None:
     """
     kwargs = {}
 
-    # Take care of thread ownership if in-memory db
     if db_url == 'sqlite://':
         kwargs.update({
-            'connect_args': {'check_same_thread': False},
             'poolclass': StaticPool,
-            'echo': False,
+        })
+    # Take care of thread ownership
+    if db_url.startswith('sqlite://'):
+        kwargs.update({
+            'connect_args': {'check_same_thread': False},
         })
 
     try:
-        engine = create_engine(db_url, **kwargs)
+        engine = create_engine(db_url, future=True, **kwargs)
     except NoSuchModuleError:
         raise OperationalException(f"Given value for db_url: '{db_url}' "
                                    f"is no valid database URL! (See {_SQL_DOCS_URL})")
@@ -58,7 +58,7 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None:
     # https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope
     # Scoped sessions proxy requests to the appropriate thread-local session.
     # We should use the scoped_session object - not a seperately initialized version
-    Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
+    Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=True))
     Trade.query = Trade._session.query_property()
     Order.query = Trade._session.query_property()
     PairLock.query = Trade._session.query_property()
@@ -77,7 +77,7 @@ def cleanup_db() -> None:
     Flushes all pending operations to disk.
     :return: None
     """
-    Trade.query.session.flush()
+    Trade.commit()
 
 
 def clean_dry_run_db() -> None:
@@ -89,6 +89,7 @@ def clean_dry_run_db() -> None:
         # Check we are updating only a dry_run order not a prod one
         if 'dry_run' in trade.open_order_id:
             trade.open_order_id = None
+    Trade.commit()
 
 
 class Order(_DECL_BASE):
@@ -112,16 +113,17 @@ class Order(_DECL_BASE):
 
     trade = relationship("Trade", back_populates="orders")
 
-    ft_order_side = Column(String, nullable=False)
-    ft_pair = Column(String, nullable=False)
+    ft_order_side = Column(String(25), nullable=False)
+    ft_pair = Column(String(25), nullable=False)
     ft_is_open = Column(Boolean, nullable=False, default=True, index=True)
 
-    order_id = Column(String, nullable=False, index=True)
-    status = Column(String, nullable=True)
-    symbol = Column(String, nullable=True)
-    order_type = Column(String, nullable=True)
-    side = Column(String, nullable=True)
+    order_id = Column(String(255), nullable=False, index=True)
+    status = Column(String(255), nullable=True)
+    symbol = Column(String(25), nullable=True)
+    order_type = Column(String(50), nullable=True)
+    side = Column(String(25), nullable=True)
     price = Column(Float, nullable=True)
+    average = Column(Float, nullable=True)
     amount = Column(Float, nullable=True)
     filled = Column(Float, nullable=True)
     remaining = Column(Float, nullable=True)
@@ -150,6 +152,7 @@ class Order(_DECL_BASE):
         self.price = order.get('price', self.price)
         self.amount = order.get('amount', self.amount)
         self.filled = order.get('filled', self.filled)
+        self.average = order.get('average', self.average)
         self.remaining = order.get('remaining', self.remaining)
         self.cost = order.get('cost', self.cost)
         if 'timestamp' in order and order['timestamp'] is not None:
@@ -175,6 +178,7 @@ class Order(_DECL_BASE):
         if filtered_orders:
             oobj = filtered_orders[0]
             oobj.update_from_ccxt_object(order)
+            Order.query.session.commit()
         else:
             logger.warning(f"Did not find order for {order}.")
 
@@ -427,12 +431,13 @@ class LocalTrade():
         elif order_type in ('stop_loss_limit', 'stop-loss', 'stop-loss-limit', 'stop'):
             self.stoploss_order_id = None
             self.close_rate_requested = self.stop_loss
+            self.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value
             if self.is_open:
                 logger.info(f'{order_type.upper()} is hit for {self}.')
             self.close(safe_value_fallback(order, 'average', 'price'))
         else:
             raise ValueError(f'Unknown order type: {order_type}')
-        cleanup_db()
+        Trade.commit()
 
     def close(self, rate: float, *, show_msg: bool = True) -> None:
         """
@@ -567,23 +572,6 @@ class LocalTrade():
         else:
             return None
 
-    @staticmethod
-    def get_trades(trade_filter=None) -> Query:
-        """
-        Helper function to query Trades using filters.
-        :param trade_filter: Optional filter to apply to trades
-                             Can be either a Filter object, or a List of filters
-                             e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])`
-                             e.g. `(trade_filter=Trade.id == trade_id)`
-        :return: unsorted query object
-        """
-        if trade_filter is not None:
-            if not isinstance(trade_filter, list):
-                trade_filter = [trade_filter]
-            return Trade.query.filter(*trade_filter)
-        else:
-            return Trade.query
-
     @staticmethod
     def get_trades_proxy(*, pair: str = None, is_open: bool = None,
                          open_date: datetime = None, close_date: datetime = None,
@@ -636,83 +624,7 @@ class LocalTrade():
         """
         Query trades from persistence layer
         """
-        return Trade.get_trades(Trade.is_open.is_(True)).all()
-
-    @staticmethod
-    def get_open_order_trades():
-        """
-        Returns all open trades
-        """
-        return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
-
-    @staticmethod
-    def get_open_trades_without_assigned_fees():
-        """
-        Returns all open trades which don't have open fees set correctly
-        """
-        return Trade.get_trades([Trade.fee_open_currency.is_(None),
-                                 Trade.orders.any(),
-                                 Trade.is_open.is_(True),
-                                 ]).all()
-
-    @staticmethod
-    def get_sold_trades_without_assigned_fees():
-        """
-        Returns all closed trades which don't have fees set correctly
-        """
-        return Trade.get_trades([Trade.fee_close_currency.is_(None),
-                                 Trade.orders.any(),
-                                 Trade.is_open.is_(False),
-                                 ]).all()
-
-    @staticmethod
-    def total_open_trades_stakes() -> float:
-        """
-        Calculates total invested amount in open trades
-        in stake currency
-        """
-        if Trade.use_db:
-            total_open_stake_amount = Trade.query.with_entities(
-                func.sum(Trade.stake_amount)).filter(Trade.is_open.is_(True)).scalar()
-        else:
-            total_open_stake_amount = sum(
-                t.stake_amount for t in Trade.get_trades_proxy(is_open=True))
-        return total_open_stake_amount or 0
-
-    @staticmethod
-    def get_overall_performance() -> List[Dict[str, Any]]:
-        """
-        Returns List of dicts containing all Trades, including profit and trade count
-        """
-        pair_rates = Trade.query.with_entities(
-            Trade.pair,
-            func.sum(Trade.close_profit).label('profit_sum'),
-            func.count(Trade.pair).label('count')
-        ).filter(Trade.is_open.is_(False))\
-            .group_by(Trade.pair) \
-            .order_by(desc('profit_sum')) \
-            .all()
-        return [
-            {
-                'pair': pair,
-                'profit': rate,
-                'count': count
-            }
-            for pair, rate, count in pair_rates
-        ]
-
-    @staticmethod
-    def get_best_pair():
-        """
-        Get best pair with closed trade.
-        :returns: Tuple containing (pair, profit_sum)
-        """
-        best_pair = Trade.query.with_entities(
-            Trade.pair, func.sum(Trade.close_profit).label('profit_sum')
-        ).filter(Trade.is_open.is_(False)) \
-            .group_by(Trade.pair) \
-            .order_by(desc('profit_sum')).first()
-        return best_pair
+        return Trade.get_trades_proxy(is_open=True)
 
     @staticmethod
     def stoploss_reinitialization(desired_stoploss):
@@ -749,15 +661,15 @@ class Trade(_DECL_BASE, LocalTrade):
 
     orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan")
 
-    exchange = Column(String, nullable=False)
-    pair = Column(String, nullable=False, index=True)
+    exchange = Column(String(25), nullable=False)
+    pair = Column(String(25), nullable=False, index=True)
     is_open = Column(Boolean, nullable=False, default=True, index=True)
     fee_open = Column(Float, nullable=False, default=0.0)
     fee_open_cost = Column(Float, nullable=True)
-    fee_open_currency = Column(String, nullable=True)
+    fee_open_currency = Column(String(25), nullable=True)
     fee_close = Column(Float, nullable=False, default=0.0)
     fee_close_cost = Column(Float, nullable=True)
-    fee_close_currency = Column(String, nullable=True)
+    fee_close_currency = Column(String(25), nullable=True)
     open_rate = Column(Float)
     open_rate_requested = Column(Float)
     # open_trade_value - calculated via _calc_open_trade_value
@@ -771,7 +683,7 @@ class Trade(_DECL_BASE, LocalTrade):
     amount_requested = Column(Float)
     open_date = Column(DateTime, nullable=False, default=datetime.utcnow)
     close_date = Column(DateTime)
-    open_order_id = Column(String)
+    open_order_id = Column(String(255))
     # absolute value of the stop loss
     stop_loss = Column(Float, nullable=True, default=0.0)
     # percentage value of the stop loss
@@ -781,16 +693,16 @@ class Trade(_DECL_BASE, LocalTrade):
     # percentage value of the initial stop loss
     initial_stop_loss_pct = Column(Float, nullable=True)
     # stoploss order id which is on exchange
-    stoploss_order_id = Column(String, nullable=True, index=True)
+    stoploss_order_id = Column(String(255), nullable=True, index=True)
     # last update time of the stoploss order on exchange
     stoploss_last_update = Column(DateTime, nullable=True)
     # absolute value of the highest reached price
     max_rate = Column(Float, nullable=True, default=0.0)
     # Lowest price reached
     min_rate = Column(Float, nullable=True)
-    sell_reason = Column(String, nullable=True)
-    sell_order_status = Column(String, nullable=True)
-    strategy = Column(String, nullable=True)
+    sell_reason = Column(String(100), nullable=True)
+    sell_order_status = Column(String(100), nullable=True)
+    strategy = Column(String(100), nullable=True)
     timeframe = Column(Integer, nullable=True)
 
     def __init__(self, **kwargs):
@@ -803,14 +715,18 @@ class Trade(_DECL_BASE, LocalTrade):
             Order.query.session.delete(order)
 
         Trade.query.session.delete(self)
-        Trade.query.session.flush()
+        Trade.commit()
+
+    @staticmethod
+    def commit():
+        Trade.query.session.commit()
 
     @staticmethod
     def get_trades_proxy(*, pair: str = None, is_open: bool = None,
                          open_date: datetime = None, close_date: datetime = None,
                          ) -> List['LocalTrade']:
         """
-        Helper function to query Trades.
+        Helper function to query Trades.j
         Returns a List of trades, filtered on the parameters given.
         In live mode, converts the filter to a database query and returns all rows
         In Backtest mode, uses filters on Trade.trades to get the result.
@@ -835,6 +751,109 @@ class Trade(_DECL_BASE, LocalTrade):
                 close_date=close_date
             )
 
+    @staticmethod
+    def get_trades(trade_filter=None) -> Query:
+        """
+        Helper function to query Trades using filters.
+        NOTE: Not supported in Backtesting.
+        :param trade_filter: Optional filter to apply to trades
+                             Can be either a Filter object, or a List of filters
+                             e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])`
+                             e.g. `(trade_filter=Trade.id == trade_id)`
+        :return: unsorted query object
+        """
+        if not Trade.use_db:
+            raise NotImplementedError('`Trade.get_trades()` not supported in backtesting mode.')
+        if trade_filter is not None:
+            if not isinstance(trade_filter, list):
+                trade_filter = [trade_filter]
+            return Trade.query.filter(*trade_filter)
+        else:
+            return Trade.query
+
+    @staticmethod
+    def get_open_order_trades():
+        """
+        Returns all open trades
+        NOTE: Not supported in Backtesting.
+        """
+        return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
+
+    @staticmethod
+    def get_open_trades_without_assigned_fees():
+        """
+        Returns all open trades which don't have open fees set correctly
+        NOTE: Not supported in Backtesting.
+        """
+        return Trade.get_trades([Trade.fee_open_currency.is_(None),
+                                 Trade.orders.any(),
+                                 Trade.is_open.is_(True),
+                                 ]).all()
+
+    @staticmethod
+    def get_sold_trades_without_assigned_fees():
+        """
+        Returns all closed trades which don't have fees set correctly
+        NOTE: Not supported in Backtesting.
+        """
+        return Trade.get_trades([Trade.fee_close_currency.is_(None),
+                                 Trade.orders.any(),
+                                 Trade.is_open.is_(False),
+                                 ]).all()
+
+    @staticmethod
+    def total_open_trades_stakes() -> float:
+        """
+        Calculates total invested amount in open trades
+        in stake currency
+        """
+        if Trade.use_db:
+            total_open_stake_amount = Trade.query.with_entities(
+                func.sum(Trade.stake_amount)).filter(Trade.is_open.is_(True)).scalar()
+        else:
+            total_open_stake_amount = sum(
+                t.stake_amount for t in LocalTrade.get_trades_proxy(is_open=True))
+        return total_open_stake_amount or 0
+
+    @staticmethod
+    def get_overall_performance() -> List[Dict[str, Any]]:
+        """
+        Returns List of dicts containing all Trades, including profit and trade count
+        NOTE: Not supported in Backtesting.
+        """
+        pair_rates = Trade.query.with_entities(
+            Trade.pair,
+            func.sum(Trade.close_profit).label('profit_sum'),
+            func.sum(Trade.close_profit_abs).label('profit_sum_abs'),
+            func.count(Trade.pair).label('count')
+        ).filter(Trade.is_open.is_(False))\
+            .group_by(Trade.pair) \
+            .order_by(desc('profit_sum_abs')) \
+            .all()
+        return [
+            {
+                'pair': pair,
+                'profit': profit,
+                'profit_abs': profit_abs,
+                'count': count
+            }
+            for pair, profit, profit_abs, count in pair_rates
+        ]
+
+    @staticmethod
+    def get_best_pair():
+        """
+        Get best pair with closed trade.
+        NOTE: Not supported in Backtesting.
+        :returns: Tuple containing (pair, profit_sum)
+        """
+        best_pair = Trade.query.with_entities(
+            Trade.pair, func.sum(Trade.close_profit).label('profit_sum')
+        ).filter(Trade.is_open.is_(False)) \
+            .group_by(Trade.pair) \
+            .order_by(desc('profit_sum')).first()
+        return best_pair
+
 
 class PairLock(_DECL_BASE):
     """
@@ -844,8 +863,8 @@ class PairLock(_DECL_BASE):
 
     id = Column(Integer, primary_key=True)
 
-    pair = Column(String, nullable=False, index=True)
-    reason = Column(String, nullable=True)
+    pair = Column(String(25), nullable=False, index=True)
+    reason = Column(String(255), nullable=True)
     # Time the pair was locked (start time)
     lock_time = Column(DateTime, nullable=False)
     # Time until the pair is locked (end time)

freqtrade/persistence/pairlock_middleware.py

@@ -49,7 +49,7 @@ class PairLocks():
         )
         if PairLocks.use_db:
             PairLock.query.session.add(lock)
-            PairLock.query.session.flush()
+            PairLock.query.session.commit()
         else:
             PairLocks.locks.append(lock)
 
@@ -99,7 +99,7 @@ class PairLocks():
         for lock in locks:
             lock.active = False
         if PairLocks.use_db:
-            PairLock.query.session.flush()
+            PairLock.query.session.commit()
 
     @staticmethod
     def is_global_lock(now: Optional[datetime] = None) -> bool:

freqtrade/plot/plotting.py

@@ -47,7 +47,7 @@ def init_plotscript(config, markets: List, startup_candles: int = 0):
     data = load_data(
         datadir=config.get('datadir'),
         pairs=pairs,
-        timeframe=config.get('timeframe', '5m'),
+        timeframe=config['timeframe'],
         timerange=timerange,
         startup_candles=startup_candles,
         data_format=config.get('dataformat_ohlcv', 'json'),
@@ -56,7 +56,7 @@ def init_plotscript(config, markets: List, startup_candles: int = 0):
     if startup_candles and data:
         min_date, max_date = get_timerange(data)
         logger.info(f"Loading data from {min_date} to {max_date}")
-        timerange.adjust_start_if_necessary(timeframe_to_seconds(config.get('timeframe', '5m')),
+        timerange.adjust_start_if_necessary(timeframe_to_seconds(config['timeframe']),
                                             startup_candles, min_date)
 
     no_trades = False
@@ -77,7 +77,8 @@ def init_plotscript(config, markets: List, startup_candles: int = 0):
         )
     except ValueError as e:
         raise OperationalException(e) from e
-    trades = trim_dataframe(trades, timerange, 'open_date')
+    if not trades.empty:
+        trades = trim_dataframe(trades, timerange, 'open_date')
 
     return {"ohlcv": data,
             "trades": trades,
@@ -95,20 +96,34 @@ def add_indicators(fig, row, indicators: Dict[str, Dict], data: pd.DataFrame) ->
                        Dict key must correspond to dataframe column.
     :param data: candlestick DataFrame
     """
+    plot_kinds = {
+        'scatter': go.Scatter,
+        'bar': go.Bar,
+    }
     for indicator, conf in indicators.items():
         logger.debug(f"indicator {indicator} with config {conf}")
         if indicator in data:
             kwargs = {'x': data['date'],
                       'y': data[indicator].values,
-                      'mode': 'lines',
                       'name': indicator
                       }
-            if 'color' in conf:
-                kwargs.update({'line': {'color': conf['color']}})
-            scatter = go.Scatter(
-                **kwargs
-            )
-            fig.add_trace(scatter, row, 1)
+
+            plot_type = conf.get('type', 'scatter')
+            color = conf.get('color')
+            if plot_type == 'bar':
+                kwargs.update({'marker_color': color or 'DarkSlateGrey',
+                               'marker_line_color': color or 'DarkSlateGrey'})
+            else:
+                if color:
+                    kwargs.update({'line': {'color': color}})
+                kwargs['mode'] = 'lines'
+                if plot_type != 'scatter':
+                    logger.warning(f'Indicator {indicator} has unknown plot trace kind {plot_type}'
+                                   f', assuming "scatter".')
+
+            kwargs.update(conf.get('plotly', {}))
+            trace = plot_kinds[plot_type](**kwargs)
+            fig.add_trace(trace, row, 1)
         else:
             logger.info(
                 'Indicator "%s" ignored. Reason: This indicator is not found '
@@ -273,8 +288,8 @@ def plot_area(fig, row: int, data: pd.DataFrame, indicator_a: str,
     :param fig: Plot figure to append to
     :param row: row number for this plot
     :param data: candlestick DataFrame
-    :param indicator_a: indicator name as populated in stragetie
-    :param indicator_b: indicator name as populated in stragetie
+    :param indicator_a: indicator name as populated in strategy
+    :param indicator_b: indicator name as populated in strategy
     :param label: label for the filled area
     :param fill_color: color to be used for the filled area
     :return: fig with added  filled_traces plot
@@ -540,8 +555,11 @@ def load_and_plot_trades(config: Dict[str, Any]):
 
         df_analyzed = strategy.analyze_ticker(data, {'pair': pair})
         df_analyzed = trim_dataframe(df_analyzed, timerange)
-        trades_pair = trades.loc[trades['pair'] == pair]
-        trades_pair = extract_trades_of_period(df_analyzed, trades_pair)
+        if not trades.empty:
+            trades_pair = trades.loc[trades['pair'] == pair]
+            trades_pair = extract_trades_of_period(df_analyzed, trades_pair)
+        else:
+            trades_pair = trades
 
         fig = generate_candlestick_graph(
             pair=pair,
@@ -565,6 +583,9 @@ def plot_profit(config: Dict[str, Any]) -> None:
     But should be somewhat proportional, and therefor useful
     in helping out to find a good algorithm.
     """
+    if 'timeframe' not in config:
+        raise OperationalException('Timeframe must be set in either config or via --timeframe.')
+
     exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
     plot_elements = init_plotscript(config, list(exchange.markets))
     trades = plot_elements['trades']
@@ -581,7 +602,8 @@ def plot_profit(config: Dict[str, Any]) -> None:
     # Create an average close price of all the pairs that were involved.
     # this could be useful to gauge the overall market trend
     fig = generate_profit_graph(plot_elements['pairs'], plot_elements['ohlcv'],
-                                trades, config.get('timeframe', '5m'),
+                                trades, config['timeframe'],
                                 config.get('stake_currency', ''))
     store_plot_file(fig, filename='freqtrade-profit-plot.html',
-                    directory=config['user_data_dir'] / 'plot', auto_open=True)
+                    directory=config['user_data_dir'] / 'plot',
+                    auto_open=config.get('plot_auto_open', False))

freqtrade/plugins/pairlist/AgeFilter.py

@@ -71,14 +71,14 @@ class AgeFilter(IPairList):
                 daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None
                 if not self._validate_pair_loc(p, daily_candles):
                     pairlist.remove(p)
-        logger.info(f"Validated {len(pairlist)} pairs.")
+        self.log_once(f"Validated {len(pairlist)} pairs.", logger.info)
         return pairlist
 
     def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool:
         """
         Validate age for the ticker
         :param pair: Pair that's currently validated
-        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param ticker: ticker dict as returned from ccxt.fetch_tickers()
         :return: True if the pair can stay, false if it should be removed
         """
         # Check symbol in cache
@@ -86,7 +86,7 @@ class AgeFilter(IPairList):
             return True
 
         if daily_candles is not None:
-            if len(daily_candles) > self._min_days_listed:
+            if len(daily_candles) >= self._min_days_listed:
                 # We have fetched at least the minimum required number of daily candles
                 # Add to cache, store the time we last checked this symbol
                 self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000

freqtrade/plugins/pairlist/IPairList.py

@@ -7,7 +7,7 @@ from copy import deepcopy
 from typing import Any, Dict, List
 
 from freqtrade.exceptions import OperationalException
-from freqtrade.exchange import market_is_active
+from freqtrade.exchange import Exchange, market_is_active
 from freqtrade.mixins import LoggingMixin
 
 
@@ -16,7 +16,7 @@ logger = logging.getLogger(__name__)
 
 class IPairList(LoggingMixin, ABC):
 
-    def __init__(self, exchange, pairlistmanager,
+    def __init__(self, exchange: Exchange, pairlistmanager,
                  config: Dict[str, Any], pairlistconfig: Dict[str, Any],
                  pairlist_pos: int) -> None:
         """
@@ -28,7 +28,7 @@ class IPairList(LoggingMixin, ABC):
         """
         self._enabled = True
 
-        self._exchange = exchange
+        self._exchange: Exchange = exchange
         self._pairlistmanager = pairlistmanager
         self._config = config
         self._pairlistconfig = pairlistconfig
@@ -68,7 +68,7 @@ class IPairList(LoggingMixin, ABC):
         filter_pairlist() method.
 
         :param pair: Pair that's currently validated
-        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param ticker: ticker dict as returned from ccxt.fetch_tickers()
         :return: True if the pair can stay, false if it should be removed
         """
         raise NotImplementedError()

freqtrade/plugins/pairlist/PerformanceFilter.py

@@ -19,7 +19,7 @@ class PerformanceFilter(IPairList):
     def needstickers(self) -> bool:
         """
         Boolean property defining if tickers are necessary.
-        If no Pairlist requries tickers, an empty List is passed
+        If no Pairlist requires tickers, an empty List is passed
         as tickers argument to filter_pairlist
         """
         return False
@@ -39,7 +39,12 @@ class PerformanceFilter(IPairList):
         :return: new allowlist
         """
         # Get the trading performance for pairs from database
-        performance = pd.DataFrame(Trade.get_overall_performance())
+        try:
+            performance = pd.DataFrame(Trade.get_overall_performance())
+        except AttributeError:
+            # Performancefilter does not work in backtesting.
+            self.log_once("PerformanceFilter is not available in this mode.", logger.warning)
+            return pairlist
 
         # Skip performance-based sorting if no performance data is available
         if len(performance) == 0:

freqtrade/plugins/pairlist/PrecisionFilter.py

@@ -48,7 +48,7 @@ class PrecisionFilter(IPairList):
         Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very
         low value pairs.
         :param pair: Pair that's currently validated
-        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param ticker: ticker dict as returned from ccxt.fetch_tickers()
         :return: True if the pair can stay, false if it should be removed
         """
         stop_price = ticker['ask'] * self._stoploss

freqtrade/plugins/pairlist/PriceFilter.py

@@ -27,9 +27,13 @@ class PriceFilter(IPairList):
         self._max_price = pairlistconfig.get('max_price', 0)
         if self._max_price < 0:
             raise OperationalException("PriceFilter requires max_price to be >= 0")
+        self._max_value = pairlistconfig.get('max_value', 0)
+        if self._max_value < 0:
+            raise OperationalException("PriceFilter requires max_value to be >= 0")
         self._enabled = ((self._low_price_ratio > 0) or
                          (self._min_price > 0) or
-                         (self._max_price > 0))
+                         (self._max_price > 0) or
+                         (self._max_value > 0))
 
     @property
     def needstickers(self) -> bool:
@@ -51,6 +55,8 @@ class PriceFilter(IPairList):
             active_price_filters.append(f"below {self._min_price:.8f}")
         if self._max_price != 0:
             active_price_filters.append(f"above {self._max_price:.8f}")
+        if self._max_value != 0:
+            active_price_filters.append(f"Value above {self._max_value:.8f}")
 
         if len(active_price_filters):
             return f"{self.name} - Filtering pairs priced {' or '.join(active_price_filters)}."
@@ -61,7 +67,7 @@ class PriceFilter(IPairList):
         """
         Check if if one price-step (pip) is > than a certain barrier.
         :param pair: Pair that's currently validated
-        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param ticker: ticker dict as returned from ccxt.fetch_tickers()
         :return: True if the pair can stay, false if it should be removed
         """
         if ticker.get('last', None) is None or ticker.get('last') == 0:
@@ -79,6 +85,32 @@ class PriceFilter(IPairList):
                               f"because 1 unit is {changeperc * 100:.3f}%", logger.info)
                 return False
 
+        # Perform low_amount check
+        if self._max_value != 0:
+            price = ticker['last']
+            market = self._exchange.markets[pair]
+            limits = market['limits']
+            if ('amount' in limits and 'min' in limits['amount']
+                    and limits['amount']['min'] is not None):
+                min_amount = limits['amount']['min']
+                min_precision = market['precision']['amount']
+
+                min_value = min_amount * price
+                if self._exchange.precisionMode == 4:
+                    # tick size
+                    next_value = (min_amount + min_precision) * price
+                else:
+                    # Decimal places
+                    min_precision = pow(0.1, min_precision)
+                    next_value = (min_amount + min_precision) * price
+                diff = next_value - min_value
+
+                if diff > self._max_value:
+                    self.log_once(f"Removed {pair} from whitelist, "
+                                  f"because min value change of {diff} > {self._max_value}.",
+                                  logger.info)
+                    return False
+
         # Perform min_price check.
         if self._min_price != 0:
             if ticker['last'] < self._min_price:
@@ -89,7 +121,7 @@ class PriceFilter(IPairList):
         # Perform max_price check.
         if self._max_price != 0:
             if ticker['last'] > self._max_price:
-                self.log_once(f"Removed {ticker['symbol']} from whitelist, "
+                self.log_once(f"Removed {pair} from whitelist, "
                               f"because last price > {self._max_price:.8f}", logger.info)
                 return False