Merge branch 'develop' into develop

.devcontainer/Dockerfile

@@ -1,20 +1,21 @@
 FROM freqtradeorg/freqtrade:develop
 
+USER root
 # Install dependencies
 COPY requirements-dev.txt /freqtrade/
+
 RUN apt-get update \
-    && apt-get -y install git mercurial sudo vim \
+    && apt-get -y install git mercurial sudo vim build-essential \
     && apt-get clean \
-    && pip install autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir \
-    && useradd -u 1000 -U -m ftuser \
     && mkdir -p /home/ftuser/.vscode-server /home/ftuser/.vscode-server-insiders /home/ftuser/commandhistory \
     && echo "export PROMPT_COMMAND='history -a'" >> /home/ftuser/.bashrc \
     && echo "export HISTFILE=~/commandhistory/.bash_history" >> /home/ftuser/.bashrc \
-    && mv /root/.local /home/ftuser/.local/ \
     && chown ftuser:ftuser -R /home/ftuser/.local/ \
     && chown ftuser: -R /home/ftuser/
 
 USER ftuser
 
+RUN pip install --user autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir
+
 # Empty the ENTRYPOINT to allow all commands
 ENTRYPOINT []

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,6 @@
+---
+blank_issues_enabled: false
+contact_links:
+  - name: Discord Server
+    url: https://discord.gg/MA9v74M
+    about: Ask a question or get community support from our Discord server

.github/workflows/ci.yml

@@ -148,6 +148,7 @@ jobs:
 
     - name: Installation - macOS
       run: |
+        brew update
         brew install hdf5 c-blosc
         python -m pip install --upgrade pip
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
@@ -300,7 +301,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Cleanup previous runs on this branch
-      uses: rokroskar/workflow-run-cleanup-action@v0.2.2
+      uses: rokroskar/workflow-run-cleanup-action@v0.3.3
       if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/stable' && github.repository == 'freqtrade/freqtrade'"
       env:
         GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9.4-slim-buster as base
+FROM python:3.9.5-slim-buster as base
 
 # Setup env
 ENV LANG C.UTF-8

README.md

@@ -1,4 +1,4 @@
-# Freqtrade
+# ![freqtrade](https://raw.githubusercontent.com/freqtrade/freqtrade/develop/docs/assets/freqtrade_poweredby.svg)
 
 [![Freqtrade CI](https://github.com/freqtrade/freqtrade/workflows/Freqtrade%20CI/badge.svg)](https://github.com/freqtrade/freqtrade/actions/)
 [![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop)
@@ -154,7 +154,7 @@ You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/
 If you discover a bug in the bot, please
 [search our issue tracker](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue)
 first. If it hasn't been reported, please
-[create a new issue](https://github.com/freqtrade/freqtrade/issues/new) and
+[create a new issue](https://github.com/freqtrade/freqtrade/issues/new/choose) and
 ensure you follow the template guide so that our team can assist you as
 quickly as possible.
 
@@ -163,7 +163,7 @@ quickly as possible.
 Have you a great idea to improve the bot you want to share? Please,
 first search if this feature was not [already discussed](https://github.com/freqtrade/freqtrade/labels/enhancement).
 If it hasn't been requested, please
-[create a new request](https://github.com/freqtrade/freqtrade/issues/new)
+[create a new request](https://github.com/freqtrade/freqtrade/issues/new/choose)
 and ensure you follow the template guide so that it does not get lost
 in the bug reports.
 

build_helpers/install_ta-lib.sh

@@ -8,10 +8,13 @@ if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then
   tar zxvf ta-lib-0.4.0-src.tar.gz
   cd ta-lib \
   && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \
+  && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' -o config.guess \
+  && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub \
   && ./configure --prefix=${INSTALL_LOC}/ \
-  && make \
+  && make -j$(nproc) \
   && which sudo && sudo make install || make install \
   && cd ..
 else
   echo "TA-lib already installed, skipping installation"
 fi
+#  && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \

build_helpers/install_windows.ps1

@@ -1,16 +1,15 @@
 # Downloads don't work automatically, since the URL is regenerated via javascript.
 # Downloaded from https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib
-# Invoke-WebRequest -Uri "https://download.lfd.uci.edu/pythonlibs/xxxxxxx/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl" -OutFile "TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl"
 
 python -m pip install --upgrade pip
 
 $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"
 
 if ($pyv -eq '3.7') {
-    pip install build_helpers\TA_Lib-0.4.19-cp37-cp37m-win_amd64.whl
+    pip install build_helpers\TA_Lib-0.4.20-cp37-cp37m-win_amd64.whl
 }
 if ($pyv -eq '3.8') {
-    pip install build_helpers\TA_Lib-0.4.19-cp38-cp38-win_amd64.whl
+    pip install build_helpers\TA_Lib-0.4.20-cp38-cp38-win_amd64.whl
 }
 
 pip install -r requirements-dev.txt

config_ftx.json.example

@@ -0,0 +1,99 @@
+{
+    "max_open_trades": 3,
+    "stake_currency": "USD",
+    "stake_amount": 50,
+    "tradable_balance_ratio": 0.99,
+    "fiat_display_currency": "USD",
+    "timeframe": "5m",
+    "dry_run": true,
+    "cancel_open_orders_on_exit": false,
+    "unfilledtimeout": {
+        "buy": 10,
+        "sell": 30
+    },
+    "bid_strategy": {
+        "ask_last_balance": 0.0,
+        "use_order_book": false,
+        "order_book_top": 1,
+        "check_depth_of_market": {
+            "enabled": false,
+            "bids_to_ask_delta": 1
+        }
+    },
+    "ask_strategy": {
+        "use_order_book": false,
+        "order_book_min": 1,
+        "order_book_max": 1,
+        "use_sell_signal": true,
+        "sell_profit_only": false,
+        "ignore_roi_if_buy_signal": false
+    },
+    "exchange": {
+        "name": "ftx",
+        "key": "your_exchange_key",
+        "secret": "your_exchange_secret",
+        "ccxt_config": {"enableRateLimit": true},
+        "ccxt_async_config": {
+            "enableRateLimit": true,
+            "rateLimit": 50
+        },
+        "pair_whitelist": [
+            "BTC/USD",
+            "ETH/USD",
+            "BNB/USD",
+            "USDT/USD",
+            "LTC/USD",
+            "SRM/USD",
+            "SXP/USD",
+            "XRP/USD",
+            "DOGE/USD",
+            "1INCH/USD",
+            "CHZ/USD",
+            "MATIC/USD",
+            "LINK/USD",
+            "OXY/USD",
+            "SUSHI/USD"
+        ],
+        "pair_blacklist": [
+            "FTT/USD"
+        ]
+    },
+    "pairlists": [
+        {"method": "StaticPairList"}
+    ],
+    "edge": {
+        "enabled": false,
+        "process_throttle_secs": 3600,
+        "calculate_since_number_of_days": 7,
+        "allowed_risk": 0.01,
+        "stoploss_range_min": -0.01,
+        "stoploss_range_max": -0.1,
+        "stoploss_range_step": -0.01,
+        "minimum_winrate": 0.60,
+        "minimum_expectancy": 0.20,
+        "min_trade_number": 10,
+        "max_trade_duration_minute": 1440,
+        "remove_pumps": false
+    },
+    "telegram": {
+        "enabled": false,
+        "token": "your_telegram_token",
+        "chat_id": "your_telegram_chat_id"
+    },
+    "api_server": {
+        "enabled": false,
+        "listen_ip_address": "127.0.0.1",
+        "listen_port": 8080,
+        "verbosity": "error",
+        "jwt_secret_key": "somethingrandom",
+        "CORS_origins": [],
+        "username": "freqtrader",
+        "password": "SuperSecurePassword"
+    },
+    "bot_name": "freqtrade",
+    "initial_state": "running",
+    "forcebuy_enable": false,
+    "internals": {
+        "process_throttle_secs": 5
+    }
+}

config_full.json.example

@@ -23,7 +23,8 @@
     "stoploss": -0.10,
     "unfilledtimeout": {
         "buy": 10,
-        "sell": 30
+        "sell": 30,
+        "unit": "minutes"
     },
     "bid_strategy": {
         "price_side": "bid",
@@ -163,7 +164,9 @@
             "warning": "on",
             "startup": "on",
             "buy": "on",
+            "buy_fill": "on",
             "sell": "on",
+            "sell_fill": "on",
             "buy_cancel": "on",
             "sell_cancel": "on"
         }

docker/Dockerfile.aarch64

@@ -0,0 +1,58 @@
+FROM --platform=linux/arm64/v8 python:3.9.4-slim-buster as base
+
+# Setup env
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONFAULTHANDLER 1
+ENV PATH=/home/ftuser/.local/bin:$PATH
+ENV FT_APP_ENV="docker"
+
+# Prepare environment
+RUN mkdir /freqtrade \
+  && apt-get update \
+  && apt-get -y install libatlas3-base curl sqlite3 libhdf5-serial-dev sudo \
+  && apt-get clean \
+  && useradd -u 1000 -G sudo -U -m ftuser \
+  && chown ftuser:ftuser /freqtrade \
+  # Allow sudoers
+  && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers
+
+WORKDIR /freqtrade
+
+# Install dependencies
+FROM base as python-deps
+RUN apt-get update \
+  && apt-get -y install curl build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \
+  && apt-get clean \
+  && pip install --upgrade pip
+
+# Install TA-lib
+COPY build_helpers/* /tmp/
+RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib*
+ENV LD_LIBRARY_PATH /usr/local/lib
+
+# Install dependencies
+COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/
+USER ftuser
+RUN  pip install --user --no-cache-dir numpy \
+  && pip install --user --no-cache-dir -r requirements-hyperopt.txt
+
+# Copy dependencies to runtime-image
+FROM base as runtime-image
+COPY --from=python-deps /usr/local/lib /usr/local/lib
+ENV LD_LIBRARY_PATH /usr/local/lib
+
+COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local
+
+USER ftuser
+# Install and execute
+COPY --chown=ftuser:ftuser . /freqtrade/
+
+RUN pip install -e . --user --no-cache-dir \
+  && mkdir /freqtrade/user_data/ \
+  && freqtrade install-ui
+
+ENTRYPOINT ["freqtrade"]
+# Default to trade mode
+CMD [ "trade" ]

docs/advanced-hyperopt.md

@@ -79,9 +79,31 @@ class MyAwesomeStrategy(IStrategy):
     class HyperOpt:
         # Define a custom stoploss space.
         def stoploss_space(self):
-            return [Real(-0.05, -0.01, name='stoploss')]
+            return [SKDecimal(-0.05, -0.01, decimals=3, name='stoploss')]
 ```
 
+## Space options
+
+For the additional spaces, scikit-optimize (in combination with Freqtrade) provides the following space types:
+
+* `Categorical` - Pick from a list of categories (e.g. `Categorical(['a', 'b', 'c'], name="cat")`)
+* `Integer` - Pick from a range of whole numbers (e.g. `Integer(1, 10, name='rsi')`)
+* `SKDecimal` - Pick from a range of decimal numbers with limited precision (e.g. `SKDecimal(0.1, 0.5, decimals=3, name='adx')`). *Available only with freqtrade*.
+* `Real` - Pick from a range of decimal numbers with full precision (e.g. `Real(0.1, 0.5, name='adx')`
+
+You can import all of these from `freqtrade.optimize.space`, although `Categorical`, `Integer` and `Real` are only aliases for their corresponding scikit-optimize Spaces. `SKDecimal` is provided by freqtrade for faster optimizations.
+
+``` python
+from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal, Real  # noqa
+```
+
+!!! Hint "SKDecimal vs. Real"
+    We recommend to use `SKDecimal` instead of the `Real` space in almost all cases. While the Real space provides full accuracy (up to ~16 decimal places) - this precision is rarely needed, and leads to unnecessary long hyperopt times.
+
+    Assuming the definition of a rather small space (`SKDecimal(0.10, 0.15, decimals=2, name='xxx')`) - SKDecimal will have 5 possibilities (`[0.10, 0.11, 0.12, 0.13, 0.14, 0.15]`).
+
+    A corresponding real space `Real(0.10, 0.15 name='xxx')`  on the other hand has an almost unlimited number of possibilities (`[0.10, 0.010000000001, 0.010000000002, ... 0.014999999999, 0.01500000000]`).
+
 ---
 
 ## Legacy Hyperopt

docs/assets/ccxt-logo.svg

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid meet" viewBox="0 0 90 90" width="100" height="100"><defs><path d="M0 90L0 0L90 0L90 90L0 90ZM50 60L60 60L60 80L70 80L70 60L80 60L80 50L50 50L50 60ZM30 80L40 80L40 70L30 70L30 80ZM30 60L20 60L20 70L10 70L10 80L20 80L20 70L30 70L30 60L40 60L40 50L30 50L30 60ZM10 60L20 60L20 50L10 50L10 60ZM10 40L40 40L40 30L20 30L20 20L40 20L40 10L10 10L10 40ZM50 40L80 40L80 30L60 30L60 20L80 20L80 10L50 10L50 40Z" id="c6g67PWSoP"></path></defs><g><g><g><use xlink:href="#c6g67PWSoP" opacity="1" fill="#000000" fill-opacity="1"></use></g></g></g></svg>

docs/backtesting.md

@@ -15,7 +15,8 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                              [--data-format-ohlcv {json,jsongz,hdf5}]
                              [--max-open-trades INT]
                              [--stake-amount STAKE_AMOUNT] [--fee FLOAT]
-                             [--eps] [--dmmp] [--enable-protections]
+                             [-p PAIRS [PAIRS ...]] [--eps] [--dmmp]
+                             [--enable-protections]
                              [--dry-run-wallet DRY_RUN_WALLET]
                              [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
                              [--export EXPORT] [--export-filename PATH]
@@ -37,6 +38,9 @@ optional arguments:
                         setting.
   --fee FLOAT           Specify fee ratio. Will be applied twice (on trade
                         entry and exit).
+  -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
+                        Limit command to these pairs. Pairs are space-
+                        separated.
   --eps, --enable-position-stacking
                         Allow buying the same pair multiple times (position
                         stacking).
@@ -233,29 +237,29 @@ The most important in the backtesting is to understand the result.
 A backtesting result will look like that:
 
 ```
-========================================================= BACKTESTING REPORT ========================================================
+========================================================= BACKTESTING REPORT ==========================================================
-| Pair     |   Buys |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Wins |  Draws |  Losses |
+| Pair     |   Buys |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration |  Wins Draws Loss   Win%  |
-|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|--------:|
+|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:-------------|-------------------------:|
-| ADA/BTC  |     35 |          -0.11 |          -3.88 |      -0.00019428 |          -1.94 | 4:35:00        |    14 |      0 |      21 |
+| ADA/BTC  |     35 |          -0.11 |          -3.88 |      -0.00019428 |          -1.94 | 4:35:00      |    14     0    21   40.0 |
-| ARK/BTC  |     11 |          -0.41 |          -4.52 |      -0.00022647 |          -2.26 | 2:03:00        |     3 |      0 |       8 |
+| ARK/BTC  |     11 |          -0.41 |          -4.52 |      -0.00022647 |          -2.26 | 2:03:00      |     3     0     8   27.3 |
-| BTS/BTC  |     32 |           0.31 |           9.78 |       0.00048938 |           4.89 | 5:05:00        |    18 |      0 |      14 |
+| BTS/BTC  |     32 |           0.31 |           9.78 |       0.00048938 |           4.89 | 5:05:00      |    18     0    14   56.2 |
-| DASH/BTC |     13 |          -0.08 |          -1.07 |      -0.00005343 |          -0.53 | 4:39:00        |     6 |      0 |       7 |
+| DASH/BTC |     13 |          -0.08 |          -1.07 |      -0.00005343 |          -0.53 | 4:39:00      |     6     0     7   46.2 |
-| ENG/BTC  |     18 |           1.36 |          24.54 |       0.00122807 |          12.27 | 2:50:00        |     8 |      0 |      10 |
+| ENG/BTC  |     18 |           1.36 |          24.54 |       0.00122807 |          12.27 | 2:50:00      |     8     0    10   44.4 |
-| EOS/BTC  |     36 |           0.08 |           3.06 |       0.00015304 |           1.53 | 3:34:00        |    16 |      0 |      20 |
+| EOS/BTC  |     36 |           0.08 |           3.06 |       0.00015304 |           1.53 | 3:34:00      |    16     0    20   44.4 |
-| ETC/BTC  |     26 |           0.37 |           9.51 |       0.00047576 |           4.75 | 6:14:00        |    11 |      0 |      15 |
+| ETC/BTC  |     26 |           0.37 |           9.51 |       0.00047576 |           4.75 | 6:14:00      |    11     0    15   42.3 |
-| ETH/BTC  |     33 |           0.30 |           9.96 |       0.00049856 |           4.98 | 7:31:00        |    16 |      0 |      17 |
+| ETH/BTC  |     33 |           0.30 |           9.96 |       0.00049856 |           4.98 | 7:31:00      |    16     0    17   48.5 |
-| IOTA/BTC |     32 |           0.03 |           1.09 |       0.00005444 |           0.54 | 3:12:00        |    14 |      0 |      18 |
+| IOTA/BTC |     32 |           0.03 |           1.09 |       0.00005444 |           0.54 | 3:12:00      |    14     0    18   43.8 |
-| LSK/BTC  |     15 |           1.75 |          26.26 |       0.00131413 |          13.13 | 2:58:00        |     6 |      0 |       9 |
+| LSK/BTC  |     15 |           1.75 |          26.26 |       0.00131413 |          13.13 | 2:58:00      |     6     0     9   40.0 |
-| LTC/BTC  |     32 |          -0.04 |          -1.38 |      -0.00006886 |          -0.69 | 4:49:00        |    11 |      0 |      21 |
+| LTC/BTC  |     32 |          -0.04 |          -1.38 |      -0.00006886 |          -0.69 | 4:49:00      |    11     0    21   34.4 |
-| NANO/BTC |     17 |           1.26 |          21.39 |       0.00107058 |          10.70 | 1:55:00        |    10 |      0 |       7 |
+| NANO/BTC |     17 |           1.26 |          21.39 |       0.00107058 |          10.70 | 1:55:00      |    10     0     7   58.5 |
-| NEO/BTC  |     23 |           0.82 |          18.97 |       0.00094936 |           9.48 | 2:59:00        |    10 |      0 |      13 |
+| NEO/BTC  |     23 |           0.82 |          18.97 |       0.00094936 |           9.48 | 2:59:00      |    10     0    13   43.5 |
-| REQ/BTC  |      9 |           1.17 |          10.54 |       0.00052734 |           5.27 | 3:47:00        |     4 |      0 |       5 |
+| REQ/BTC  |      9 |           1.17 |          10.54 |       0.00052734 |           5.27 | 3:47:00      |     4     0     5   44.4 |
-| XLM/BTC  |     16 |           1.22 |          19.54 |       0.00097800 |           9.77 | 3:15:00        |     7 |      0 |       9 |
+| XLM/BTC  |     16 |           1.22 |          19.54 |       0.00097800 |           9.77 | 3:15:00      |     7     0     9   43.8 |
-| XMR/BTC  |     23 |          -0.18 |          -4.13 |      -0.00020696 |          -2.07 | 5:30:00        |    12 |      0 |      11 |
+| XMR/BTC  |     23 |          -0.18 |          -4.13 |      -0.00020696 |          -2.07 | 5:30:00      |    12     0    11   52.2 |
-| XRP/BTC  |     35 |           0.66 |          22.96 |       0.00114897 |          11.48 | 3:49:00        |    12 |      0 |      23 |
+| XRP/BTC  |     35 |           0.66 |          22.96 |       0.00114897 |          11.48 | 3:49:00      |    12     0    23   34.3 |
-| ZEC/BTC  |     22 |          -0.46 |         -10.18 |      -0.00050971 |          -5.09 | 2:22:00        |     7 |      0 |      15 |
+| ZEC/BTC  |     22 |          -0.46 |         -10.18 |      -0.00050971 |          -5.09 | 2:22:00      |     7     0    15   31.8 |
-| TOTAL    |    429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00        |   186 |      0 |     243 |
+| TOTAL    |    429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00      |   186     0   243   43.4 |
-========================================================= SELL REASON STATS =========================================================
+========================================================= SELL REASON STATS ==========================================================
 | Sell Reason        |   Sells |  Wins |  Draws |  Losses |
 |:-------------------|--------:|------:|-------:|--------:|
 | trailing_stop_loss |     205 |   150 |      0 |      55 |
@@ -263,11 +267,11 @@ A backtesting result will look like that:
 | sell_signal        |      56 |    36 |      0 |      20 |
 | force_sell         |       2 |     0 |      0 |       2 |
 ====================================================== LEFT OPEN TRADES REPORT ======================================================
-| Pair     |   Buys |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Wins |  Draws |  Losses |
+| Pair     |   Buys |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Win Draw Loss Win% |
-|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|--------:|
+|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|--------------------:|
-| ADA/BTC  |      1 |           0.89 |           0.89 |       0.00004434 |           0.44 | 6:00:00        |     1 |      0 |       0 |
+| ADA/BTC  |      1 |           0.89 |           0.89 |       0.00004434 |           0.44 | 6:00:00        |    1    0    0  100 |
-| LTC/BTC  |      1 |           0.68 |           0.68 |       0.00003421 |           0.34 | 2:00:00        |     1 |      0 |       0 |
+| LTC/BTC  |      1 |           0.68 |           0.68 |       0.00003421 |           0.34 | 2:00:00        |    1    0    0  100 |
-| TOTAL    |      2 |           0.78 |           1.57 |       0.00007855 |           0.78 | 4:00:00        |     2 |      0 |       0 |
+| TOTAL    |      2 |           0.78 |           1.57 |       0.00007855 |           0.78 | 4:00:00        |    2    0    0  100 |
 =============== SUMMARY METRICS ===============
 | Metric                | Value               |
 |-----------------------+---------------------|
@@ -293,6 +297,8 @@ A backtesting result will look like that:
 | Days win/draw/lose    | 12 / 82 / 25        |
 | Avg. Duration Winners | 4:23:00             |
 | Avg. Duration Loser   | 6:55:00             |
+| Zero Duration Trades  | 4.6% (20)           |
+| Rejected Buy signals  | 3089                |
 |                       |                     |
 | Min balance           | 0.00945123 BTC      |
 | Max balance           | 0.01846651 BTC      |
@@ -314,7 +320,7 @@ The last line will give you the overall performance of your strategy,
 here:
 
 ```
-| TOTAL    |         429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00        |      186 |    243 |
+| TOTAL    |    429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00      |   186     0   243   43.4 |
 ```
 
 The bot has made `429` trades for an average duration of `4:12:00`, with a performance of `76.20%` (profit), that means it has
@@ -380,6 +386,8 @@ It contains some useful key metrics about performance of your strategy on backte
 | Days win/draw/lose    | 12 / 82 / 25        |
 | Avg. Duration Winners | 4:23:00             |
 | Avg. Duration Loser   | 6:55:00             |
+| Zero Duration Trades  | 4.6% (20)           |
+| Rejected Buy signals  | 3089                |
 |                       |                     |
 | Min balance           | 0.00945123 BTC      |
 | Max balance           | 0.01846651 BTC      |
@@ -409,6 +417,8 @@ It contains some useful key metrics about performance of your strategy on backte
 - `Best day` / `Worst day`: Best and worst day based on daily profit.
 - `Days win/draw/lose`: Winning / Losing days (draws are usually days without closed trade).
 - `Avg. Duration Winners` / `Avg. Duration Loser`: Average durations for winning and losing trades.
+- `Zero Duration Trades`: A number of trades that completed within same candle as they opened and had `trailing_stop_loss` sell reason. A significant amount of such trades may indicate that strategy is exploiting trailing stoploss behavior in backtesting and produces unrealistic results.
+- `Rejected Buy signals`: Buy signals that could not be acted upon due to max_open_trades being reached.
 - `Min balance` / `Max balance`: Lowest and Highest Wallet balance during the backtest period.
 - `Drawdown`: Maximum drawdown experienced. For example, the value of 50% means that from highest to subsequent lowest point, a 50% drop was experienced).
 - `Drawdown high` / `Drawdown low`: Profit at the beginning and end of the largest drawdown period. A negative low value means initial capital lost.
@@ -468,11 +478,11 @@ There will be an additional table comparing win/losses of the different strategi
 Detailed output for all strategies one after the other will be available, so make sure to scroll up to see the details per strategy.
 
 ```
-=========================================================== STRATEGY SUMMARY ===========================================================
+=========================================================== STRATEGY SUMMARY =========================================================================
-| Strategy    |   Buys |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Wins |  Draws | Losses |
+| Strategy    |   Buys |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Wins |  Draws | Losses | Drawdown % |
-|:------------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|-------:|
+|:------------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|-------:|-----------:|
-| Strategy1   |    429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00        |   186 |      0 |    243 |
+| Strategy1   |    429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00        |   186 |      0 |    243 |       45.2 |
-| Strategy2   |   1487 |          -0.13 |        -197.58 |      -0.00988917 |         -98.79 | 4:43:00        |   662 |      0 |    825 |
+| Strategy2   |   1487 |          -0.13 |        -197.58 |      -0.00988917 |         -98.79 | 4:43:00        |   662 |      0 |    825 |     241.68 |
 ```
 
 ## Next step

docs/configuration.md

@@ -11,7 +11,16 @@ Per default, the bot loads the configuration from the `config.json` file, locate
 
 You can specify a different configuration file used by the bot with the `-c/--config` command line option.
 
-In some advanced use cases, multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream.
+Multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream.
+
+!!! Tip "Use multiple configuration files to keep secrets secret"
+    You can use a 2nd configuration file containing your secrets. That way you can share your "primary" configuration file, while still keeping your API keys for yourself.
+
+    ``` bash
+    freqtrade trade --config user_data/config.json --config user_data/config-private.json <...>
+    ```
+    The 2nd file should only specify what you intend to override.
+    If a key is in more than one of the configurations, then the "last specified configuration" wins (in the above example, `config-private.json`).
 
 If you used the [Quick start](installation.md/#quick-start) method for installing 
 the bot, the installation script should have already created the default configuration file (`config.json`) for you.
@@ -59,8 +68,9 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md#trailing-stop-loss-only-once-the-trade-has-reached-a-certain-offset). [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `0.0` (no offset).* <br> **Datatype:** Float
 | `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.*  <br> **Datatype:** Boolean
 | `fee` | Fee used during backtesting / dry-runs. Should normally not be configured, which has freqtrade fall back to the exchange default fee. Set as ratio (e.g. 0.001 = 0.1%). Fee is applied twice for each trade, once when buying, once when selling. <br> **Datatype:** Float (as ratio)
-| `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
+| `unfilledtimeout.buy` | **Required.** How long (in minutes or seconds) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
-| `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
+| `unfilledtimeout.sell` | **Required.** How long (in minutes or seconds) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
+| `unfilledtimeout.unit` | Unit to use in unfilledtimeout setting. Note: If you set unfilledtimeout.unit to "seconds", "internals.process_throttle_secs" must be inferior or equal to timeout [Strategy Override](#parameters-in-the-strategy). <br> *Defaults to `minutes`.* <br> **Datatype:** String
 | `bid_strategy.price_side` | Select the side of the spread the bot should look at to get the buy rate. [More information below](#buy-price-side).<br> *Defaults to `bid`.* <br> **Datatype:** String (either `ask` or `bid`).
 | `bid_strategy.ask_last_balance` | **Required.** Interpolate the bidding price. More information [below](#buy-price-without-orderbook-enabled).
 | `bid_strategy.use_order_book` | Enable buying using the rates in [Order Book Bids](#buy-price-with-orderbook-enabled). <br> **Datatype:** Boolean
@@ -167,7 +177,7 @@ This exchange has also a limit on USD - where all orders must be > 10$ - which h
 
 To guarantee safe execution, freqtrade will not allow buying with a stake-amount of 10.1$, instead, it'll make sure that there's enough space to place a stoploss below the pair (+ an offset, defined by `amount_reserve_percent`, which defaults to 5%).
 
-With a stoploss of 10% - we'd therefore end up with a value of ~13.8$ (`12 * (1 + 0.05 + 0.1)`).
+With a reserve of 5%, the minimum stake amount would be ~12.6$ (`12 * (1 + 0.05)`). If we take in account a stoploss of 10% on top of that - we'd end up with a value of ~14$ (`12.6 / (1 - 0.1)`).
 
 To limit this calculation in case of large stoploss values, the calculated minimum stake-limit will never be more than 50% above the real limit.
 
@@ -518,16 +528,27 @@ API Keys are usually only required for live trading (trading for real money, bot
 **Insert your Exchange API key (change them by fake api keys):**
 
 ```json
+{
     "exchange": {
         "name": "bittrex",
         "key": "af8ddd35195e9dc500b9a6f799f6f5c93d89193b",
         "secret": "08a9dc6db3d7b53e1acebd9275677f4b0a04f1a5",
-        ...
+        //"password": "", // Optional, not needed by all exchanges)
+        // ...
+    }
+    //...
 }
 ```
 
 You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange.
 
+!!! Hint "Keep your secrets secret"
+    To keep your secrets secret, we recommend to use a 2nd configuration for your API keys.
+    Simply use the above snippet in a new configuration file (e.g. `config-private.json`) and keep your settings in this file.
+    You can then start the bot with `freqtrade trade --config user_data/config.json --config user_data/config-private.json <...>` to have your keys loaded.
+
+    **NEVER** share your private configuration file or your exchange keys with anyone!
+
 ### Using proxy with Freqtrade
 
 To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration.

docs/data-download.md

@@ -11,8 +11,9 @@ Otherwise `--exchange` becomes mandatory.
 You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101-`). For incremental downloads, the relative approach should be used.
 
 !!! Tip "Tip: Updating existing data"
-    If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data.
+    If you already have backtesting data available in your data-directory and would like to refresh this data up to today, do not use `--days` or `--timerange` parameters. Freqtrade will keep the available data and only download the missing data.
-    Be careful though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded.
+    If you are updating existing data after inserting new pairs that you have no data for, use `--new-pairs-days xx` parameter. Specified number of days will be downloaded for new pairs while old pairs will be updated with missing data only.
+    If you use `--days xx` parameter alone - data for specified number of days will be downloaded for _all_ pairs. Be careful, if specified number of days is smaller than gap between now and last downloaded candle - freqtrade will delete all existing data to avoid gaps in candle data.
 
 ### Usage
 
@@ -20,8 +21,9 @@ You can use a relative timerange (`--days 20`) or an absolute starting point (`-
 usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                [-d PATH] [--userdir PATH]
                                [-p PAIRS [PAIRS ...]] [--pairs-file FILE]
-                               [--days INT] [--timerange TIMERANGE]
+                               [--days INT] [--new-pairs-days INT]
-                               [--dl-trades] [--exchange EXCHANGE]
+                               [--timerange TIMERANGE] [--dl-trades]
+                               [--exchange EXCHANGE]
                                [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
                                [--erase]
                                [--data-format-ohlcv {json,jsongz,hdf5}]
@@ -30,10 +32,12 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
 optional arguments:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
-                        Show profits for only these pairs. Pairs are space-
+                        Limit command to these pairs. Pairs are space-
                         separated.
   --pairs-file FILE     File containing a list of pairs to download.
   --days INT            Download data for given number of days.
+  --new-pairs-days INT  Download data of new pairs for given number of days.
+                        Default: `None`.
   --timerange TIMERANGE
                         Specify what timerange of data to use.
   --dl-trades           Download trades instead of OHLCV data. The bot will
@@ -48,10 +52,10 @@ optional arguments:
                         exchange/pairs/timeframes.
   --data-format-ohlcv {json,jsongz,hdf5}
                         Storage format for downloaded candle (OHLCV) data.
-                        (default: `json`).
+                        (default: `None`).
   --data-format-trades {json,jsongz,hdf5}
                         Storage format for downloaded trades data. (default:
-                        `jsongz`).
+                        `None`).
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).

docs/docker_quickstart.md

@@ -10,11 +10,11 @@ Start by downloading and installing Docker CE for your platform:
 * [Windows](https://docs.docker.com/docker-for-windows/install/)
 * [Linux](https://docs.docker.com/install/)
 
-To simplify running freqtrade, please install [`docker-compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the below [docker quick start guide](#docker-quick-start).
+To simplify running freqtrade, [`docker-compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the below [docker quick start guide](#docker-quick-start).
 
 ## Freqtrade with docker-compose
 
-Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) ready for usage.
+Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/stable/docker-compose.yml) ready for usage.
 
 !!! Note
     - The following section assumes that `docker` and `docker-compose` are installed and available to the logged in user.
@@ -22,7 +22,7 @@ Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.co
 
 ### Docker quick start
 
-Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory.
+Create a new directory and place the [docker-compose file](https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml) in this directory.
 
 === "PC/MAC/Linux"
     ``` bash
@@ -48,6 +48,8 @@ Create a new directory and place the [docker-compose file](https://github.com/fr
     # Download the docker-compose file from the repository
     curl https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml -o docker-compose.yml
 
+    # Edit the compose file to use an image named `*_pi` (stable_pi or develop_pi)
+
     # Pull the freqtrade image
     docker-compose pull
 
@@ -65,6 +67,40 @@ Create a new directory and place the [docker-compose file](https://github.com/fr
         # image: freqtradeorg/freqtrade:develop_pi
         ```
 
+=== "ARM 64 Systenms (Mac M1, Raspberry Pi 4, Jetson Nano)"
+    In case of a Mac M1, make sure that your docker installation is running in native mode
+    Arm64 images are not yet provided via Docker Hub and need to be build locally first.
+    Depending on the device, this may take a few minutes (Apple M1) or multiple hours (Raspberry Pi)
+
+    ``` bash
+    # Clone Freqtrade repository
+    git clone  https://github.com/freqtrade/freqtrade.git
+    cd freqtrade
+    # Optionally switch to the stable version
+    git checkout stable   
+
+    # Modify your docker-compose file to enable building and change the image name 
+    # (see the Note Box below for necessary changes)
+
+    # Build image
+    docker-compose build
+    
+    # Create user directory structure
+    docker-compose run --rm freqtrade create-userdir --userdir user_data
+
+    # Create configuration - Requires answering interactive questions
+    docker-compose run --rm freqtrade new-config --config user_data/config.json
+    ```
+
+    !!! Note "Change your docker Image"
+        You have to change the docker image in the docker-compose file for your arm64 build to work properly.
+        ``` yml
+        image: freqtradeorg/freqtrade:custom_arm64
+        build:
+          context: .
+          dockerfile: "./docker/Dockerfile.aarch64"
+        ```
+
 The above snippet creates a new directory called `ft_userdata`, downloads the latest compose file and pulls the freqtrade image.
 The last 2 steps in the snippet create the directory with `user_data`, as well as (interactively) the default configuration based on your selections.
 

docs/edge.md

@@ -215,8 +215,10 @@ Let's say the stake currency is **ETH** and there is $10$ **ETH** on the wallet.
 usage: freqtrade edge [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
                       [--userdir PATH] [-s NAME] [--strategy-path PATH]
                       [-i TIMEFRAME] [--timerange TIMERANGE]
+                      [--data-format-ohlcv {json,jsongz,hdf5}]
                       [--max-open-trades INT] [--stake-amount STAKE_AMOUNT]
-                      [--fee FLOAT] [--stoplosses STOPLOSS_RANGE]
+                      [--fee FLOAT] [-p PAIRS [PAIRS ...]]
+                      [--stoplosses STOPLOSS_RANGE]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -224,6 +226,9 @@ optional arguments:
                         Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).
   --timerange TIMERANGE
                         Specify what timerange of data to use.
+  --data-format-ohlcv {json,jsongz,hdf5}
+                        Storage format for downloaded candle (OHLCV) data.
+                        (default: `None`).
   --max-open-trades INT
                         Override the value of the `max_open_trades`
                         configuration setting.
@@ -232,6 +237,9 @@ optional arguments:
                         setting.
   --fee FLOAT           Specify fee ratio. Will be applied twice (on trade
                         entry and exit).
+  -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
+                        Limit command to these pairs. Pairs are space-
+                        separated.
   --stoplosses STOPLOSS_RANGE
                         Defines a range of stoploss values against which edge
                         will assess the strategy. The format is "min,max,step"

docs/exchanges.md

@@ -7,10 +7,10 @@ This page combines common gotchas and informations which are exchange-specific a
 !!! Tip "Stoploss on Exchange"
     Binance supports `stoploss_on_exchange` and uses stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it.
 
-### Blacklists
+### Binance Blacklist
 
 For Binance, please add `"BNB/<STAKE>"` to your blacklist to avoid issues.
-Accounts having BNB accounts use this to pay for fees - if your first trade happens to be on `BNB`, further trades will consume this position and make the initial BNB order unsellable as the expected amount is not there anymore.
+Accounts having BNB accounts use this to pay for fees - if your first trade happens to be on `BNB`, further trades will consume this position and make the initial BNB trade unsellable as the expected amount is not there anymore.
 
 ### Binance sites
 
@@ -100,6 +100,23 @@ To use subaccounts with FTX, you need to edit the configuration and add the foll
 }
 ```
 
+## Kucoin
+
+Kucoin requries a passphrase for each api key, you will therefore need to add this key into the configuration so your exchange section looks as follows:
+
+```json
+"exchange": {
+    "name": "kucoin",
+    "key": "your_exchange_key",
+    "secret": "your_exchange_secret",
+    "password": "your_exchange_api_key_password",
+```
+
+### Kucoin Blacklists
+
+For Kucoin, please add `"KCS/<STAKE>"` to your blacklist to avoid issues.
+Accounts having KCS accounts use this to pay for fees - if your first trade happens to be on `KCS`, further trades will consume this position and make the initial KCS trade unsellable as the expected amount is not there anymore.
+
 ## All exchanges
 
 Should you experience constant errors with Nonce (like `InvalidNonce`), it is best to regenerate the API keys. Resetting Nonce is difficult and it's usually easier to regenerate the API keys.

docs/faq.md

@@ -156,7 +156,7 @@ freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossD
 
 ### Why does it take a long time to run hyperopt?
 
-* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/X89cVG). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you.
+* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/MA9v74M). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you.
 
 * If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers:
 

docs/hyperopt.md

@@ -44,8 +44,9 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
                           [--data-format-ohlcv {json,jsongz,hdf5}]
                           [--max-open-trades INT]
                           [--stake-amount STAKE_AMOUNT] [--fee FLOAT]
-                          [--hyperopt NAME] [--hyperopt-path PATH] [--eps]
+                          [-p PAIRS [PAIRS ...]] [--hyperopt NAME]
-                          [--dmmp] [--enable-protections]
+                          [--hyperopt-path PATH] [--eps] [--dmmp]
+                          [--enable-protections]
                           [--dry-run-wallet DRY_RUN_WALLET] [-e INT]
                           [--spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]]
                           [--print-all] [--no-color] [--print-json] [-j JOBS]
@@ -69,6 +70,9 @@ optional arguments:
                         setting.
   --fee FLOAT           Specify fee ratio. Will be applied twice (on trade
                         entry and exit).
+  -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
+                        Limit command to these pairs. Pairs are space-
+                        separated.
   --hyperopt NAME       Specify hyperopt class name which will be used by the
                         bot.
   --hyperopt-path PATH  Specify additional lookup path for Hyperopt and
@@ -105,7 +109,8 @@ optional arguments:
                         reproducible hyperopt results.
   --min-trades INT      Set minimal desired number of trades for evaluations
                         in the hyperopt optimization path (default: 1).
-  --hyperopt-loss NAME  Specify the class name of the hyperopt loss function
+  --hyperopt-loss NAME, --hyperoptloss NAME
+                        Specify the class name of the hyperopt loss function
                         class (IHyperOptLoss). Different functions can
                         generate completely different results, since the
                         target for optimization is different. Built-in
@@ -160,11 +165,22 @@ Rarely you may also need to create a [nested class](advanced-hyperopt.md#overrid
 !!! Tip "Quickly optimize ROI, stoploss and trailing stoploss"
     You can quickly optimize the spaces `roi`, `stoploss` and `trailing` without changing anything in your strategy.
 
-    ```python
+    ``` bash
     # Have a working strategy at hand.
     freqtrade hyperopt --hyperopt-loss SharpeHyperOptLossDaily --spaces roi stoploss trailing --strategy MyWorkingStrategy --config config.json -e 100
     ```
 
+### Hyperopt execution logic
+
+Hyperopt will first load your data into memory and will then run `populate_indicators()` once per Pair to generate all indicators.
+
+Hyperopt will then spawn into different processes (number of processors, or `-j <n>`), and run backtesting over and over again, changing the parameters that are part of the `--spaces` defined.
+
+For every new set of parameters, freqtrade will run first `populate_buy_trend()` followed by `populate_sell_trend()`, and then run the regular backtesting process to simulate trades.
+
+After backtesting, the results are passed into the [loss function](#loss-functions), which will evaluate if this result was better or worse than previous results.  
+Based on the loss function result, hyperopt will determine the next set of parameters to try in the next round of backtesting.
+
 ### Configure your Guards and Triggers
 
 There are two places you need to change in your strategy file to add a new buy hyperopt for testing:
@@ -182,68 +198,62 @@ There you have two different types of indicators: 1. `guards` and 2. `triggers`.
     However, this guide will make this distinction to make it clear that signals should not be "sticking".
     Sticking signals are signals that are active for multiple candles. This can lead into buying a signal late (right before the signal disappears - which means that the chance of success is a lot lower than right at the beginning).
 
-Hyper-optimization will, for each epoch round, pick one trigger and possibly
+Hyper-optimization will, for each epoch round, pick one trigger and possibly multiple guards.
-multiple guards. The constructed strategy will be something like "*buy exactly when close price touches lower Bollinger band, BUT only if
-ADX > 10*".
-
-```python
-from freqtrade.strategy import IntParameter, IStrategy
-
-class MyAwesomeStrategy(IStrategy):
-    # If parameter is prefixed with `buy_` or `sell_` then specifying `space` parameter is optional
-    # and space is inferred from parameter name.
-    buy_adx_min = IntParameter(0, 100, default=10)
-
-    def populate_buy_trend(self, dataframe: 'DataFrame', metadata: dict) -> 'DataFrame':
-        dataframe.loc[
-            (
-                (dataframe['adx'] > self.buy_adx_min.value)
-            ), 'buy'] = 1
-        return dataframe
-```
 
 #### Sell optimization
 
 Similar to the buy-signal above, sell-signals can also be optimized.
 Place the corresponding settings into the following methods
 
-* Define the parameters at the class level hyperopt shall be optimizing.
+* Define the parameters at the class level hyperopt shall be optimizing, either naming them `sell_*`, or by explicitly defining `space='sell'`.
 * Within `populate_sell_trend()` - use defined parameter values instead of raw constants.
 
 The configuration and rules are the same than for buy signals.
 
+## Solving a Mystery
+
+Let's say you are curious: should you use MACD crossings or lower Bollinger Bands to trigger your buys. 
+And you also wonder should you use RSI or ADX to help with those buy decisions. 
+If you decide to use RSI or ADX, which values should I use for them? 
+
+So let's use hyperparameter optimization to solve this mystery.
+
+### Defining indicators to be used
+
+We start by calculating the indicators our strategy is going to use.
+
 ``` python
 class MyAwesomeStrategy(IStrategy):
-    # There is no strict parameter naming scheme. If you do not use `buy_` or `sell_` prefixes - 
-    # please specify to which space parameter belongs using `space` parameter. Possible values: 
-    # 'buy' or 'sell'.
-    adx_max = IntParameter(0, 100, default=50, space='sell')
 
-    def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
-        dataframe.loc[
+        """
-            (
+        Generate all indicators used by the strategy
-                (dataframe['adx'] < self.adx_max.value)
+        """
-            ), 'buy'] = 1
+        dataframe['adx'] = ta.ADX(dataframe)
+        dataframe['rsi'] = ta.RSI(dataframe)
+        macd = ta.MACD(dataframe)
+        dataframe['macd'] = macd['macd']
+        dataframe['macdsignal'] = macd['macdsignal']
+        dataframe['macdhist'] = macd['macdhist']
+
+        bollinger = ta.BBANDS(dataframe, timeperiod=20, nbdevup=2.0, nbdevdn=2.0)
+        dataframe['bb_lowerband'] = boll['lowerband']
+        dataframe['bb_middleband'] = boll['middleband']
+        dataframe['bb_upperband'] = boll['upperband']
         return dataframe
 ```
 
-## Solving a Mystery
+### Hyperoptable parameters
 
-Let's say you are curious: should you use MACD crossings or lower Bollinger
+We continue to define hyperoptable parameters:
-Bands to trigger your buys. And you also wonder should you use RSI or ADX to
-help with those buy decisions. If you decide to use RSI or ADX, which values
-should I use for them? So let's use hyperparameter optimization to solve this
-mystery.
-
-We will start by defining hyperoptable parameters:
 
 ```python
 class MyAwesomeStrategy(IStrategy):
-    buy_adx = IntParameter(20, 40, default=30)
+    buy_adx = IntParameter(20, 40, default=30, space="buy")
-    buy_rsi = IntParameter(20, 40, default=30)
+    buy_rsi = IntParameter(20, 40, default=30, space="buy")
-    buy_adx_enabled = CategoricalParameter([True, False]),
+    buy_adx_enabled = CategoricalParameter([True, False], space="buy")
-    buy_rsi_enabled = CategoricalParameter([True, False]),
+    buy_rsi_enabled = CategoricalParameter([True, False], space="buy")
-    buy_trigger = CategoricalParameter(['bb_lower', 'macd_cross_signal']),
+    buy_trigger = CategoricalParameter(['bb_lower', 'macd_cross_signal'], space="buy")
 ```
 
 Above definition says: I have five parameters I want to randomly combine to find the best combination.
@@ -252,6 +262,10 @@ Then we have three category variables. First two are either `True` or `False`.
 We use these to either enable or disable the ADX and RSI guards.
 The last one we call `trigger` and use it to decide which buy trigger we want to use.
 
+!!! Note "Parameter space assignment"
+    Parameters must either be assigned to a variable named `buy_*` or `sell_*` - or contain `space='buy'` | `space='sell'` to be assigned to a space correctly.
+    If no parameter is available for a space, you'll receive the error that no space was found when running hyperopt.
+
 So let's write the buy strategy using these values:
 
 ```python
@@ -283,7 +297,7 @@ So let's write the buy strategy using these values:
 ```
 
 Hyperopt will now call `populate_buy_trend()` many times (`epochs`) with different value combinations.  
-It will use the given historical data and make buys based on the buy signals generated with the above function.  
+It will use the given historical data and simulate buys based on the buy signals generated with the above function.  
 Based on the results, hyperopt will tell you which parameter combination produced the best results (based on the configured [loss function](#loss-functions)).
 
 !!! Note
@@ -294,6 +308,7 @@ Based on the results, hyperopt will tell you which parameter combination produce
 ## Parameter types
 
 There are four parameter types each suited for different purposes.
+
 * `IntParameter` - defines an integral parameter with upper and lower boundaries of search space.
 * `DecimalParameter` - defines a floating point parameter with a limited number of decimals (default 3). Should be preferred instead of `RealParameter` in most cases.
 * `RealParameter` - defines a floating point parameter with upper and lower boundaries and no precision limit. Rarely used as it creates a space with a near infinite number of possibilities.
@@ -308,6 +323,90 @@ There are four parameter types each suited for different purposes.
 !!! Warning
     Hyperoptable parameters cannot be used in `populate_indicators` - as hyperopt does not recalculate indicators for each epoch, so the starting value would be used in this case.
 
+### Optimizing an indicator parameter
+
+Assuming you have a simple strategy in mind - a EMA cross strategy (2 Moving averages crossing) - and you'd like to find the ideal parameters for this strategy.
+
+``` python
+from pandas import DataFrame
+from functools import reduce
+
+import talib.abstract as ta
+
+from freqtrade.strategy import IStrategy
+from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter
+import freqtrade.vendor.qtpylib.indicators as qtpylib
+
+class MyAwesomeStrategy(IStrategy):
+    stoploss = -0.05
+    timeframe = '15m'
+    # Define the parameter spaces
+    buy_ema_short = IntParameter(3, 50, default=5)
+    buy_ema_long = IntParameter(15, 200, default=50)
+
+
+    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        """Generate all indicators used by the strategy"""
+        
+        # Calculate all ema_short values
+        for val in self.buy_ema_short.range:
+            dataframe[f'ema_short_{val}'] = ta.EMA(dataframe, timeperiod=val)
+        
+        # Calculate all ema_long values
+        for val in self.buy_ema_long.range:
+            dataframe[f'ema_long_{val}'] = ta.EMA(dataframe, timeperiod=val)
+        
+        return dataframe
+
+    def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        conditions = []
+        conditions.append(qtpylib.crossed_above(
+                dataframe[f'ema_short_{self.buy_ema_short.value}'], dataframe[f'ema_long_{self.buy_ema_long.value}']
+            ))
+
+        # Check that volume is not 0
+        conditions.append(dataframe['volume'] > 0)
+
+        if conditions:
+            dataframe.loc[
+                reduce(lambda x, y: x & y, conditions),
+                'buy'] = 1
+        return dataframe
+
+    def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        conditions = []
+        conditions.append(qtpylib.crossed_above(
+                dataframe[f'ema_long_{self.buy_ema_long.value}'], dataframe[f'ema_short_{self.buy_ema_short.value}']
+            ))
+
+        # Check that volume is not 0
+        conditions.append(dataframe['volume'] > 0)
+
+        if conditions:
+            dataframe.loc[
+                reduce(lambda x, y: x & y, conditions),
+                'sell'] = 1
+        return dataframe
+```
+
+Breaking it down:
+
+Using `self.buy_ema_short.range` will return a range object containing all entries between the Parameters low and high value.
+In this case (`IntParameter(3, 50, default=5)`), the loop would run for all numbers between 3 and 50 (`[3, 4, 5, ... 49, 50]`).
+By using this in a loop, hyperopt will generate 48 new columns (`['buy_ema_3', 'buy_ema_4', ... , 'buy_ema_50']`).
+
+Hyperopt itself will then use the selected value to create the buy and sell signals
+
+While this strategy is most likely too simple to provide consistent profit, it should serve as an example how optimize indicator parameters.
+
+!!! Note
+    `self.buy_ema_short.range` will act differently between hyperopt and other modes. For hyperopt, the above example may generate 48 new columns, however for all other modes (backtesting, dry/live), it will only generate the column for the selected value. You should therefore avoid using the resulting column with explicit values (values other than `self.buy_ema_short.value`).
+
+??? Hint "Performance tip"
+    By doing the calculation of all possible indicators in `populate_indicators()`, the calculation of the indicator happens only once for every parameter.  
+    While this may slow down the hyperopt startup speed, the overall performance will increase as the Hyperopt execution itself may pick the same value for multiple epochs (changing other values).
+    You should however try to use space ranges as small as possible. Every new column will require more memory, and every possibility hyperopt can try will increase the search space.
+
 ## Loss-functions
 
 Each hyperparameter tuning requires a target. This is usually defined as a loss function (sometimes also called objective function), which should decrease for more desirable results, and increase for bad results.
@@ -460,13 +559,13 @@ As stated in the comment, you can also use it as the value of the `minimal_roi`
 
 #### Default ROI Search Space
 
-If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the timeframe used. By default the values vary in the following ranges (for some of the most used timeframes, values are rounded to 5 digits after the decimal point):
+If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the timeframe used. By default the values vary in the following ranges (for some of the most used timeframes, values are rounded to 3 digits after the decimal point):
 
 | # step | 1m     |               | 5m       |             | 1h         |               | 1d           |               |
-| ------ | ------ | ----------------- | -------- | ----------- | ---------- | ----------------- | ------------ | ----------------- |
+| ------ | ------ | ------------- | -------- | ----------- | ---------- | ------------- | ------------ | ------------- |
-| 1      | 0      | 0.01161...0.11992 | 0        | 0.03...0.31 | 0          | 0.06883...0.71124 | 0            | 0.12178...1.25835 |
+| 1      | 0      | 0.011...0.119 | 0        | 0.03...0.31 | 0          | 0.068...0.711 | 0            | 0.121...1.258 |
-| 2      | 2...8  | 0.00774...0.04255 | 10...40  | 0.02...0.11 | 120...480  | 0.04589...0.25238 | 2880...11520 | 0.08118...0.44651 |
+| 2      | 2...8  | 0.007...0.042 | 10...40  | 0.02...0.11 | 120...480  | 0.045...0.252 | 2880...11520 | 0.081...0.446 |
-| 3      | 4...20 | 0.00387...0.01547 | 20...100 | 0.01...0.04 | 240...1200 | 0.02294...0.09177 | 5760...28800 | 0.04059...0.16237 |
+| 3      | 4...20 | 0.003...0.015 | 20...100 | 0.01...0.04 | 240...1200 | 0.022...0.091 | 5760...28800 | 0.040...0.162 |
 | 4      | 6...44 | 0.0           | 30...220 | 0.0         | 360...2640 | 0.0           | 8640...63360 | 0.0           |
 
 These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the timeframe used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the timeframe used.
@@ -477,6 +576,9 @@ Override the `roi_space()` method if you need components of the ROI tables to va
 
 A sample for these methods can be found in [sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py).
 
+!!! Note "Reduced search space"
+    To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs.
+
 ### Understand Hyperopt Stoploss results
 
 If you are optimizing stoploss values (i.e. if optimization search-space contains 'all', 'default' or 'stoploss'), your result will look as follows and include stoploss:
@@ -516,6 +618,9 @@ If you have the `stoploss_space()` method in your custom hyperopt file, remove i
 
 Override the `stoploss_space()` method and define the desired range in it if you need stoploss values to vary in other range during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py).
 
+!!! Note "Reduced search space"
+    To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs.
+
 ### Understand Hyperopt Trailing Stop results
 
 If you are optimizing trailing stop values (i.e. if optimization search-space contains 'all' or 'trailing'), your result will look as follows and include trailing stop parameters:
@@ -551,6 +656,9 @@ If you are optimizing trailing stop values, Freqtrade creates the 'trailing' opt
 
 Override the `trailing_space()` method and define the desired range in it if you need values of the trailing stop parameters to vary in other ranges during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py).
 
+!!! Note "Reduced search space"
+    To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs.
+
 ### Reproducible results
 
 The search for optimal parameters starts with a few (currently 30) random combinations in the hyperspace of parameters, random Hyperopt epochs. These random epochs are marked with an asterisk character (`*`) in the first column in the Hyperopt output.
@@ -591,6 +699,16 @@ number).
 You can also enable position stacking in the configuration file by explicitly setting
 `"position_stacking"=true`.
 
+## Out of Memory errors
+
+As hyperopt consumes a lot of memory (the complete data needs to be in memory once per parallel backtesting process), it's likely that you run into "out of memory" errors.
+To combat these, you have multiple options:
+
+* reduce the amount of pairs
+* reduce the timerange used (`--timerange <timerange>`)
+* reduce the number of parallel processes (`-j <n>`)
+* Increase the memory of your machine
+
 ## Show details of Hyperopt results
 
 After you run Hyperopt for the desired amount of epochs, you can later list all results for analysis, select only best or profitable once, and show the details for any of the epochs previously evaluated. This can be done with the `hyperopt-list` and `hyperopt-show` sub-commands. The usage of these sub-commands is described in the [Utils](utils.md#list-hyperopt-results) chapter.

docs/includes/pairlists.md

@@ -60,6 +60,8 @@ When used in the chain of Pairlist Handlers in a non-leading position (after Sta
 When used on the leading position of the chain of Pairlist Handlers, it does not consider `pair_whitelist` configuration setting, but selects the top assets from all available markets (with matching stake-currency) on the exchange.
 
 The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes).
+The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists.
+Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data.
 
 `VolumePairList` is based on the ticker data from exchange, as reported by the ccxt library:
 
@@ -90,6 +92,7 @@ This filter allows freqtrade to ignore pairs until they have been listed for at
 #### PerformanceFilter
 
 Sorts pairs by past trade performance, as follows:
+
 1. Positive performance.
 2. No closed trades yet.
 3. Negative performance.
@@ -109,6 +112,7 @@ The `PriceFilter` allows filtering of pairs by price. Currently the following pr
 
 * `min_price`
 * `max_price`
+* `max_value`
 * `low_price_ratio`
 
 The `min_price` setting removes pairs where the price is below the specified price. This is useful if you wish to avoid trading very low-priced pairs.
@@ -117,6 +121,11 @@ This option is disabled by default, and will only apply if set to > 0.
 The `max_price` setting removes pairs where the price is above the specified price. This is useful if you wish to trade only low-priced pairs.
 This option is disabled by default, and will only apply if set to > 0.
 
+The `max_value` setting removes pairs where the minimum value change is above a specified value.
+This is useful when an exchange has unbalanced limits. For example, if step-size = 1 (so you can only buy 1, or 2, or 3, but not 1.1 Coins) - and the price is pretty high (like 20$) as the coin has risen sharply since the last limit adaption.
+As a result of the above, you can only buy for 20$, or 40$ - but not for 25$.
+On exchanges that deduct fees from the receiving currency (e.g. FTX) - this can result in high value coins / amounts that are unsellable as the amount is slightly below the limit.
+
 The `low_price_ratio` setting removes pairs where a raise of 1 price unit (pip) is above the `low_price_ratio` ratio.
 This option is disabled by default, and will only apply if set to > 0.
 
@@ -190,7 +199,7 @@ If the volatility over the last 10 days is not in the range of 0.05-0.50, remove
 
 ### Full example of Pairlist Handlers
 
-The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 price unit is > 1%. Then the [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) is applied and pairs are finally shuffled with the random seed set to some predefined value.
+The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#pricefilter), filtering all assets where 1 price unit is > 1%. Then the [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) is applied and pairs are finally shuffled with the random seed set to some predefined value.
 
 ```json
 "exchange": {
@@ -201,7 +210,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets,
     {
         "method": "VolumePairList",
         "number_assets": 20,
-        "sort_key": "quoteVolume",
+        "sort_key": "quoteVolume"
     },
     {"method": "AgeFilter", "min_days_listed": 10},
     {"method": "PrecisionFilter"},

docs/index.md

@@ -1,4 +1,5 @@
-# Freqtrade
+![freqtrade](assets/freqtrade_poweredby.svg)
+
 [![Freqtrade CI](https://github.com/freqtrade/freqtrade/workflows/Freqtrade%20CI/badge.svg)](https://github.com/freqtrade/freqtrade/actions/)
 [![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop)
 [![Maintainability](https://api.codeclimate.com/v1/badges/5737e6d668200b7518ff/maintainability)](https://codeclimate.com/github/freqtrade/freqtrade/maintainability)
@@ -39,7 +40,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual,
 - [X] [Bittrex](https://bittrex.com/)
 - [X] [FTX](https://ftx.com)
 - [X] [Kraken](https://kraken.com/)
-- [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_
+- [ ] [potentially many others through <img alt="ccxt" width="30px" src="assets/ccxt-logo.svg" />](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_
 
 ### Community tested
 

docs/installation.md

@@ -60,7 +60,7 @@ OS Specific steps are listed first, the [Common](#common) section below is neces
     sudo apt-get update
 
     # install packages
-    sudo apt install -y python3-pip python3-venv python3-pandas python3-pip git
+    sudo apt install -y python3-pip python3-venv python3-pandas git
     ```
 
 === "RaspberryPi/Raspbian"
@@ -269,7 +269,7 @@ git clone https://github.com/freqtrade/freqtrade.git
 cd freqtrade      
 ```
 
-#### Freqtrade instal: Conda Environment
+#### Freqtrade install: Conda Environment
 
 Prepare conda-freqtrade environment, using file `environment.yml`, which exist in main freqtrade directory
 

docs/plotting.md

@@ -37,7 +37,7 @@ usage: freqtrade plot-dataframe [-h] [-v] [--logfile FILE] [-V] [-c PATH]
 optional arguments:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
-                        Show profits for only these pairs. Pairs are space-
+                        Limit command to these pairs. Pairs are space-
                         separated.
   --indicators1 INDICATORS1 [INDICATORS1 ...]
                         Set indicators from your strategy you want in the
@@ -90,6 +90,7 @@ Strategy arguments:
                         Specify strategy class name which will be used by the
                         bot.
   --strategy-path PATH  Specify additional strategy lookup path.
+
 ```
 
 Example:
@@ -244,7 +245,7 @@ usage: freqtrade plot-profit [-h] [-v] [--logfile FILE] [-V] [-c PATH]
 optional arguments:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
-                        Show profits for only these pairs. Pairs are space-
+                        Limit command to these pairs. Pairs are space-
                         separated.
   --timerange TIMERANGE
                         Specify what timerange of data to use.
@@ -286,6 +287,7 @@ Strategy arguments:
                         Specify strategy class name which will be used by the
                         bot.
   --strategy-path PATH  Specify additional strategy lookup path.
+
 ```
 
 The `-p/--pairs`  argument, can be used to limit the pairs that are considered for this calculation.

docs/requirements-docs.txt

@@ -1,3 +1,3 @@
-mkdocs-material==7.1.0
+mkdocs-material==7.1.5
 mdx_truly_sane_lists==1.2
-pymdown-extensions==8.1.1
+pymdown-extensions==8.2

docs/rest-api.md

@@ -71,7 +71,10 @@ If you run your bot using docker, you'll need to have the bot listen to incoming
     "api_server": {
         "enabled": true,
         "listen_ip_address": "0.0.0.0",
-        "listen_port": 8080
+        "listen_port": 8080,
+        "username": "Freqtrader",
+        "password": "SuperSecret1!",
+        //...
     },
 ```
 
@@ -106,7 +109,10 @@ By default, the script assumes `127.0.0.1` (localhost) and port `8080` to be use
     "api_server": {
         "enabled": true,
         "listen_ip_address": "0.0.0.0",
-        "listen_port": 8080
+        "listen_port": 8080,
+        "username": "Freqtrader",
+        "password": "SuperSecret1!",
+        //...
     }
 }
 ```
@@ -124,7 +130,8 @@ python3 scripts/rest_client.py --config rest_config.json <command> [optional par
 | `stop` | Stops the trader.
 | `stopbuy` | Stops the trader from opening new trades. Gracefully closes open trades according to their rules.
 | `reload_config` | Reloads the configuration file.
-| `trades` | List last trades.
+| `trades` | List last trades. Limited to 500 trades per call.
+| `trade/<tradeid>` | Get specific trade.
 | `delete_trade <trade_id>` | Remove trade from the database. Tries to close open orders. Requires manual handling of this trade on the exchange.
 | `show_config` | Shows part of the current configuration with relevant settings to operation.
 | `logs` | Shows last log messages.
@@ -181,7 +188,7 @@ count
 	Return the amount of open trades.
 
 daily
-	Return the amount of open trades.
+	Return the profits for each day, and amount of trades.
 
 delete_lock
 	Delete (disable) lock from the database.
@@ -214,7 +221,7 @@ locks
 logs
 	Show latest logs.
 
-        :param limit: Limits log messages to the last <limit> logs. No limit to get all the trades.
+        :param limit: Limits log messages to the last <limit> logs. No limit to get the entire log.
 
 pair_candles
 	Return live dataframe for <pair><timeframe>.
@@ -234,6 +241,9 @@ pair_history
 performance
 	Return the performance of the different coins.
 
+ping
+	simple ping
+
 plot_config
 	Return plot configuration if the strategy defines one.
 
@@ -270,17 +280,22 @@ strategy
 
         :param strategy: Strategy class name
 
-trades
+trade
-	Return trades history.
+	Return specific trade
 
-        :param limit: Limits trades to the X last trades. No limit to get all the trades.
+        :param trade_id: Specify which trade to get.
+
+trades
+	Return trades history, sorted by id
+
+        :param limit: Limits trades to the X last trades. Max 500 trades.
+        :param offset: Offset by this amount of trades.
 
 version
 	Return the version of the bot.
 
 whitelist
 	Show the current whitelist.
-
 ```
 
 ### OpenAPI interface

docs/sql_cheatsheet.md

@@ -19,7 +19,7 @@ The freqtrade docker image does contain sqlite3, so you can edit the database wi
 
 ``` bash
 docker-compose exec freqtrade /bin/bash
-sqlite3 <databasefile>.sqlite
+sqlite3 <database-file>.sqlite
 ```
 
 ## Open the DB
@@ -99,3 +99,32 @@ DELETE FROM trades WHERE id = 31;
 
 !!! Warning
     This will remove this trade from the database. Please make sure you got the correct id and **NEVER** run this query without the `where` clause.
+
+## Use a different database system
+
+!!! Warning
+    By using one of the below database systems, you acknowledge that you know how to manage such a system. Freqtrade will not provide any support with setup or maintenance (or backups) of the below database systems.
+
+### PostgreSQL
+
+Freqtrade supports PostgreSQL by using SQLAlchemy, which supports multiple different database systems.
+
+Installation:
+`pip install psycopg2`
+
+Usage:
+`... --db-url postgresql+psycopg2://<username>:<password>@localhost:5432/<database>`
+
+Freqtrade will automatically create the tables necessary upon startup.
+
+If you're running different instances of Freqtrade, you must either setup one database per Instance or use different users / schemas for your connections.
+
+### MariaDB / MySQL
+
+Freqtrade supports MariaDB by using SQLAlchemy, which supports multiple different database systems.
+
+Installation:
+`pip install pymysql`
+
+Usage:
+`... --db-url mysql+pymysql://<username>:<password>@localhost:3306/<database>`

docs/strategy-advanced.md

@@ -40,34 +40,79 @@ class AwesomeStrategy(IStrategy):
 !!! Note
     If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
 
-***
+## Dataframe access
 
-### Storing custom information using DatetimeIndex from `dataframe`
+You may access dataframe in various strategy functions by querying it from dataprovider.
-
-Imagine you need to store an indicator like `ATR` or `RSI` into `custom_info`. To use this in a meaningful way, you will not only need the raw data of the indicator, but probably also need to keep the right timestamps.
 
 ``` python
-import talib.abstract as ta
+from freqtrade.exchange import timeframe_to_prev_date
-class AwesomeStrategy(IStrategy):
-    # Create custom dictionary
-    custom_info = {}
 
-    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+class AwesomeStrategy(IStrategy):
-        # using "ATR" here as example
+    def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: float,
-        dataframe['atr'] = ta.ATR(dataframe)
+                           rate: float, time_in_force: str, sell_reason: str,
-        if self.dp.runmode.value in ('backtest', 'hyperopt'):
+                           current_time: 'datetime', **kwargs) -> bool:
-          # add indicator mapped to correct DatetimeIndex to custom_info
+        # Obtain pair dataframe.
-          self.custom_info[metadata['pair']] = dataframe[['date', 'atr']].copy().set_index('date')
+        dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
-        return dataframe
+
+        # Obtain last available candle. Do not use current_time to look up latest candle, because 
+        # current_time points to curret incomplete candle whose data is not available.
+        last_candle = dataframe.iloc[-1].squeeze()
+        # <...>
+
+        # In dry/live runs trade open date will not match candle open date therefore it must be 
+        # rounded.
+        trade_date = timeframe_to_prev_date(self.timeframe, trade.open_date_utc)
+        # Look up trade candle.
+        trade_candle = dataframe.loc[dataframe['date'] == trade_date]
+        # trade_candle may be empty for trades that just opened as it is still incomplete.
+        if not trade_candle.empty:
+            trade_candle = trade_candle.squeeze()
+            # <...>
 ```
 
-!!! Warning
+!!! Warning "Using .iloc[-1]"
-    The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
+    You can use `.iloc[-1]` here because `get_analyzed_dataframe()` only returns candles that backtesting is allowed to see.
+    This will not work in `populate_*` methods, so make sure to not use `.iloc[]` in that area.
+    Also, this will only work starting with version 2021.5.
+
+***
+
+## Custom sell signal
+
+It is possible to define custom sell signals, indicating that specified position should be sold. This is very useful when we need to customize sell conditions for each individual trade, or if you need the trade profit to take the sell decision. 
+
+For example you could implement a 1:2 risk-reward ROI with `custom_sell()`.
+
+Using custom_sell() signals in place of stoplosses though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange.
 
 !!! Note
-    If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
+    Returning a `string` or `True` from this method is equal to setting sell signal on a candle at specified time. This method is not called when sell signal is set already, or if sell signals are disabled (`use_sell_signal=False` or `sell_profit_only=True` while profit is below `sell_profit_offset`). `string` max length is 64 characters. Exceeding this limit will cause the message to be truncated to 64 characters.
 
-See `custom_stoploss` examples below on how to access the saved dataframe columns
+An example of how we can use different indicators depending on the current profit and also sell trades that were open longer than one day:
+
+``` python
+class AwesomeStrategy(IStrategy):
+    def custom_sell(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float,
+                    current_profit: float, **kwargs):
+        dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
+        last_candle = dataframe.iloc[-1].squeeze()
+
+        # Above 20% profit, sell when rsi < 80
+        if current_profit > 0.2:
+            if last_candle['rsi'] < 80:
+                return 'rsi_below_80'
+
+        # Between 2% and 10%, sell if EMA-long above EMA-short
+        if 0.02 < current_profit < 0.1:
+            if last_candle['emalong'] > last_candle['emashort']:
+                return 'ema_long_below_80'
+
+        # Sell any positions at a loss if they are held for more than one day.
+        if current_profit < 0.0 and (current_time - trade.open_date_utc).days >= 1:
+            return 'unclog'
+```
+
+See [Dataframe access](#dataframe-access) for more information about dataframe use in strategy callbacks.
 
 ## Custom stoploss
 
@@ -222,7 +267,6 @@ Instead of continuously trailing behind the current price, this example sets fix
 * Once profit is > 25% - set stoploss to 15% above open price.
 * Once profit is > 40% - set stoploss to 25% above open price.
 
-
 ``` python
 from datetime import datetime
 from freqtrade.persistence import Trade
@@ -248,63 +292,46 @@ class AwesomeStrategy(IStrategy):
         # return maximum stoploss value, keeping current stoploss price unchanged
         return 1
 ```
+
 #### Custom stoploss using an indicator from dataframe example
 
-Imagine you want to use `custom_stoploss()` to use a trailing indicator like e.g. "ATR"
+Absolute stoploss value may be derived from indicators stored in dataframe. Example uses parabolic SAR below the price as stoploss.
-
-See: "Storing custom information using DatetimeIndex from `dataframe`" example above) on how to store the indicator into `custom_info`
-
-!!! Warning
-    only use .iat[-1] in live mode, not in backtesting/hyperopt
-    otherwise you will look into the future
-    see [Common mistakes when developing strategies](strategy-customization.md#common-mistakes-when-developing-strategies) for more info.
 
 ``` python
-from freqtrade.persistence import Trade
-from freqtrade.state import RunMode
-
 class AwesomeStrategy(IStrategy):
 
-    # ... populate_* methods
+    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        # <...>
+        dataframe['sar'] = ta.SAR(dataframe)
 
     use_custom_stoploss = True
 
     def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
                         current_rate: float, current_profit: float, **kwargs) -> float:
 
-        result = 1
+        dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
-        if self.custom_info and pair in self.custom_info and trade:
+        last_candle = dataframe.iloc[-1].squeeze()
-            # using current_time directly (like below) will only work in backtesting.
-            # so check "runmode" to make sure that it's only used in backtesting/hyperopt
-            if self.dp and self.dp.runmode.value in ('backtest', 'hyperopt'):
-              relative_sl = self.custom_info[pair].loc[current_time]['atr']
-            # in live / dry-run, it'll be really the current time
-            else:
-              # but we can just use the last entry from an already analyzed dataframe instead
-              dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair,
-                                                                       timeframe=self.timeframe)
-              # WARNING
-              # only use .iat[-1] in live mode, not in backtesting/hyperopt
-              # otherwise you will look into the future
-              # see: https://www.freqtrade.io/en/latest/strategy-customization/#common-mistakes-when-developing-strategies
-              relative_sl = dataframe['atr'].iat[-1]
 
-            if (relative_sl is not None):
+        # Use parabolic sar as absolute stoploss price
-                # new stoploss relative to current_rate
+        stoploss_price = last_candle['sar']
-                new_stoploss = (current_rate-relative_sl)/current_rate
-                # turn into relative negative offset required by `custom_stoploss` return implementation
-                result = new_stoploss - 1
 
-        return result
+        # Convert absolute price to percentage relative to current_rate
+        if stoploss_price < current_rate:
+            return (stoploss_price / current_rate) - 1
+
+        # return maximum stoploss value, keeping current stoploss price unchanged
+        return 1
 ```
 
+See [Dataframe access](#dataframe-access) for more information about dataframe use in strategy callbacks.
+
 ---
 
 ## Custom order timeout rules
 
 Simple, time-based order-timeouts can be configured either via strategy or in the configuration in the `unfilledtimeout` section.
 
-However, freqtrade also offers a custom callback for both order types, which allows you to decide based on custom criteria if a order did time out or not.
+However, freqtrade also offers a custom callback for both order types, which allows you to decide based on custom criteria if an order did time out or not.
 
 !!! Note
     Unfilled order timeouts are not relevant during backtesting or hyperopt, and are only relevant during real (live) trading. Therefore these methods are only called in these circumstances.
@@ -530,7 +557,7 @@ Both attributes and methods may be overridden, altering behavior of the original
 
 ## Embedding Strategies
 
-Freqtrade provides you with with an easy way to embed the strategy into your configuration file.
+Freqtrade provides you with an easy way to embed the strategy into your configuration file.
 This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field,
 in your chosen config file.
 

docs/strategy-customization.md

@@ -159,7 +159,7 @@ Edit the method `populate_buy_trend()` in your strategy file to update your buy
 
 It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected.
 
-This will method will also define a new column, `"buy"`, which needs to contain 1 for buys, and 0 for "no action".
+This method will also define a new column, `"buy"`, which needs to contain 1 for buys, and 0 for "no action".
 
 Sample from `user_data/strategies/sample_strategy.py`:
 
@@ -193,7 +193,7 @@ Please note that the sell-signal is only used if `use_sell_signal` is set to tru
 
 It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected.
 
-This will method will also define a new column, `"sell"`, which needs to contain 1 for sells, and 0 for "no action".
+This method will also define a new column, `"sell"`, which needs to contain 1 for sells, and 0 for "no action".
 
 Sample from `user_data/strategies/sample_strategy.py`:
 
@@ -422,10 +422,6 @@ if self.dp:
     Returns an empty dataframe if the requested pair was not cached.
     This should not happen when using whitelisted pairs.
 
-
-!!! Warning "Warning about backtesting"
-    This method will return an empty dataframe during backtesting.
-
 ### *orderbook(pair, maximum)*
 
 ``` python
@@ -633,7 +629,7 @@ Stoploss values returned from `custom_stoploss` must specify a percentage relati
         def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
                             current_rate: float, current_profit: float, **kwargs) -> float:
 
-            # once the profit has risin above 10%, keep the stoploss at 7% above the open price
+            # once the profit has risen above 10%, keep the stoploss at 7% above the open price
             if current_profit > 0.10:
                 return stoploss_from_open(0.07, current_profit)
 

docs/strategy_analysis_example.md

@@ -195,4 +195,18 @@ graph.show(renderer="browser")
 
 ```
 
+## Plot average profit per trade as distribution graph
+
+
+```python
+import plotly.figure_factory as ff
+
+hist_data = [trades.profit_ratio]
+group_labels = ['profit_ratio']  # name of the dataset
+
+fig = ff.create_distplot(hist_data, group_labels,bin_size=0.01)
+fig.show()
+
+```
+
 Feel free to submit an issue or Pull Request enhancing this document if you would like to share ideas on how to best analyze the data.

docs/telegram-usage.md

@@ -82,12 +82,19 @@ Example configuration showing the different settings:
          "buy": "silent",
          "sell": "on",
          "buy_cancel": "silent",
-         "sell_cancel": "on"
+         "sell_cancel": "on",
+         "buy_fill": "off",
+         "sell_fill": "off"
       },
       "balance_dust_level": 0.01
    },
 ```
 
+`buy` notifications are sent when the order is placed, while `buy_fill` notifications are sent when the order is filled on the exchange.
+`sell` notifications are sent when the order is placed, while `sell_fill` notifications are sent when the order is filled on the exchange.
+`*_fill` notifications are off by default and must be explicitly enabled.
+
+
 `balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown.
 
 ## Create a custom keyboard (command shortcut buttons)
@@ -243,10 +250,14 @@ Return a summary of your profit/loss and performance.
 
 > **BITTREX:** Selling BTC/LTC with limit `0.01650000 (profit: ~-4.07%, -0.00008168)`
 
-### /forcebuy <pair>
+### /forcebuy <pair> [rate]
 
 > **BITTREX:** Buying ETH/BTC with limit `0.03400000` (`1.000000 ETH`, `225.290 USD`)
 
+Omitting the pair will open a query asking for the pair to buy (based on the current whitelist).
+
+![Telegram force-buy screenshot](assets/telegram_forcebuy.png)
+
 Note that for this to work, `forcebuy_enable` needs to be set to true.
 
 [More details](configuration.md#understand-forcebuy_enable)
@@ -255,11 +266,11 @@ Note that for this to work, `forcebuy_enable` needs to be set to true.
 
 Return the performance of each crypto-currency the bot has sold.
 > Performance:  
-> 1. `RCN/BTC 57.77%`  
+> 1. `RCN/BTC 0.003 BTC (57.77%) (1)`  
-> 2. `PAY/BTC 56.91%`  
+> 2. `PAY/BTC 0.0012 BTC (56.91%) (1)`  
-> 3. `VIB/BTC 47.07%`  
+> 3. `VIB/BTC 0.0011 BTC (47.07%) (1)`  
-> 4. `SALT/BTC 30.24%`  
+> 4. `SALT/BTC 0.0010 BTC (30.24%) (1)`  
-> 5. `STORJ/BTC 27.24%`  
+> 5. `STORJ/BTC 0.0009 BTC (27.24%) (1)`  
 > ...  
 
 ### /balance

docs/webhook-config.md

@@ -19,6 +19,11 @@ Sample configuration (tested using IFTTT).
             "value1": "Cancelling Open Buy Order for {pair}",
             "value2": "limit {limit:8f}",
             "value3": "{stake_amount:8f} {stake_currency}"
+        },
+         "webhookbuyfill": {
+            "value1": "Buy Order for {pair} filled",
+            "value2": "at {open_rate:8f}",
+            "value3": ""
         },
         "webhooksell": {
             "value1": "Selling {pair}",
@@ -30,6 +35,11 @@ Sample configuration (tested using IFTTT).
             "value2": "limit {limit:8f}",
             "value3": "profit: {profit_amount:8f} {stake_currency} ({profit_ratio})"
         },
+        "webhooksellfill": {
+            "value1": "Sell Order for {pair} filled",
+            "value2": "at {close_rate:8f}.",
+            "value3": ""
+        },
         "webhookstatus": {
             "value1": "Status: {status}",
             "value2": "",
@@ -91,6 +101,21 @@ Possible parameters are:
 * `order_type`
 * `current_rate`
 
+### Webhookbuyfill
+
+The fields in `webhook.webhookbuyfill` are filled when the bot filled a buy order. Parameters are filled using string.format.
+Possible parameters are:
+
+* `trade_id`
+* `exchange`
+* `pair`
+* `open_rate`
+* `amount`
+* `open_date`
+* `stake_amount`
+* `stake_currency`
+* `fiat_currency`
+
 ### Webhooksell
 
 The fields in `webhook.webhooksell` are filled when the bot sells a trade. Parameters are filled using string.format.
@@ -103,6 +128,27 @@ Possible parameters are:
 * `limit`
 * `amount`
 * `open_rate`
+* `profit_amount`
+* `profit_ratio`
+* `stake_currency`
+* `fiat_currency`
+* `sell_reason`
+* `order_type`
+* `open_date`
+* `close_date`
+
+### Webhooksellfill
+
+The fields in `webhook.webhooksellfill` are filled when the bot fills a sell order (closes a Trae). Parameters are filled using string.format.
+Possible parameters are:
+
+* `trade_id`
+* `exchange`
+* `pair`
+* `gain`
+* `close_rate`
+* `amount`
+* `open_rate`
 * `current_rate`
 * `profit_amount`
 * `profit_ratio`

docs/windows_installation.md

@@ -1,3 +1,5 @@
+# Windows installation
+
 We **strongly** recommend that Windows users use [Docker](docker_quickstart.md) as this will work much easier and smoother (also more secure).
 
 If that is not possible, try using the Windows Linux subsystem (WSL) - for which the Ubuntu instructions should work.
@@ -21,7 +23,7 @@ git clone https://github.com/freqtrade/freqtrade.git
 
 Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows).
 
-As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.19‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version)
+As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial pre-compiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.20‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version).
 
 Freqtrade provides these dependencies for the latest 2 Python versions (3.7 and 3.8) and for 64bit Windows.
 Other versions must be downloaded from the above link.

environment.yml

@@ -4,7 +4,7 @@ channels:
 # - defaults
 dependencies:
 # 1/4 req main
-    - python>=3.7
+    - python>=3.7,<3.9
     - numpy
     - pandas
     - pip

freqtrade/commands/arguments.py

@@ -17,7 +17,7 @@ ARGS_STRATEGY = ["strategy", "strategy_path"]
 ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"]
 
 ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
-                        "max_open_trades", "stake_amount", "fee"]
+                        "max_open_trades", "stake_amount", "fee", "pairs"]
 
 ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
                                         "enable_protections", "dry_run_wallet",
@@ -60,8 +60,9 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
 
 ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
 
-ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange",
+ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange",
-                      "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"]
+                      "download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv",
+                      "dataformat_trades"]
 
 ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
                        "db_url", "trade_source", "export", "exportfilename",

freqtrade/commands/cli_options.py

@@ -330,7 +330,7 @@ AVAILABLE_CLI_OPTIONS = {
     # Script options
     "pairs": Arg(
         '-p', '--pairs',
-        help='Show profits for only these pairs. Pairs are space-separated.',
+        help='Limit command to these pairs. Pairs are space-separated.',
         nargs='+',
     ),
     # Download data
@@ -345,6 +345,12 @@ AVAILABLE_CLI_OPTIONS = {
         type=check_int_positive,
         metavar='INT',
     ),
+    "new_pairs_days": Arg(
+        '--new-pairs-days',
+        help='Download data of new pairs for given number of days. Default: `%(default)s`.',
+        type=check_int_positive,
+        metavar='INT',
+    ),
     "download_trades": Arg(
         '--dl-trades',
         help='Download trades instead of OHLCV data. The bot will resample trades to the '

freqtrade/commands/data_commands.py

@@ -62,8 +62,8 @@ def start_download_data(args: Dict[str, Any]) -> None:
         if config.get('download_trades'):
             pairs_not_available = refresh_backtest_trades_data(
                 exchange, pairs=expanded_pairs, datadir=config['datadir'],
-                timerange=timerange, erase=bool(config.get('erase')),
+                timerange=timerange, new_pairs_days=config['new_pairs_days'],
-                data_format=config['dataformat_trades'])
+                erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
 
             # Convert downloaded trade data to different timeframes
             convert_trades_to_ohlcv(
@@ -75,8 +75,9 @@ def start_download_data(args: Dict[str, Any]) -> None:
         else:
             pairs_not_available = refresh_backtest_ohlcv_data(
                 exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
-                datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
+                datadir=config['datadir'], timerange=timerange,
-                data_format=config['dataformat_ohlcv'])
+                new_pairs_days=config['new_pairs_days'],
+                erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'])
 
     except KeyboardInterrupt:
         sys.exit("SIGINT received, aborting ...")

freqtrade/commands/hyperopt_commands.py

@@ -7,6 +7,7 @@ from colorama import init as colorama_init
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.data.btanalysis import get_latest_hyperopt_file
 from freqtrade.exceptions import OperationalException
+from freqtrade.optimize.optimize_reports import show_backtest_result
 from freqtrade.state import RunMode
 
 
@@ -125,6 +126,12 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
 
     if epochs:
         val = epochs[n]
+
+        metrics = val['results_metrics']
+        if 'strategy_name' in metrics:
+            show_backtest_result(metrics['strategy_name'], metrics,
+                                 metrics['stake_currency'])
+
         HyperoptTools.print_epoch_details(val, total_epochs, print_json, no_header,
                                           header_str="Epoch details")
 
@@ -132,11 +139,13 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
 def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
     """
     Filter our items from the list of hyperopt results
+    TODO: after 2021.5 remove all "legacy" mode queries.
     """
     if filteroptions['only_best']:
         epochs = [x for x in epochs if x['is_best']]
     if filteroptions['only_profitable']:
-        epochs = [x for x in epochs if x['results_metrics']['profit'] > 0]
+        epochs = [x for x in epochs if x['results_metrics'].get(
+            'profit', x['results_metrics'].get('profit_total', 0)) > 0]
 
     epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
 
@@ -153,34 +162,55 @@ def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
     return epochs
 
 
+def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int):
+    """
+    Filter epochs with trade-counts > trades
+    """
+    return [
+        x for x in epochs
+        if x['results_metrics'].get(
+            'trade_count', x['results_metrics'].get('total_trades', 0)
+        ) > trade_count
+    ]
+
+
 def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
 
     if filteroptions['filter_min_trades'] > 0:
-        epochs = [
+        epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades'])
-            x for x in epochs
+
-            if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades']
-        ]
     if filteroptions['filter_max_trades'] > 0:
         epochs = [
             x for x in epochs
-            if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades']
+            if x['results_metrics'].get(
+                'trade_count', x['results_metrics'].get('total_trades')
+                ) < filteroptions['filter_max_trades']
         ]
     return epochs
 
 
 def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
 
+    def get_duration_value(x):
+        # Duration in minutes ...
+        if 'duration' in x['results_metrics']:
+            return x['results_metrics']['duration']
+        else:
+            # New mode
+            avg = x['results_metrics']['holding_avg']
+            return avg.total_seconds() // 60
+
     if filteroptions['filter_min_avg_time'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
         epochs = [
             x for x in epochs
-            if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time']
+            if get_duration_value(x) > filteroptions['filter_min_avg_time']
         ]
     if filteroptions['filter_max_avg_time'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
         epochs = [
             x for x in epochs
-            if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time']
+            if get_duration_value(x) < filteroptions['filter_max_avg_time']
         ]
 
     return epochs
@@ -189,28 +219,36 @@ def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
 def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
 
     if filteroptions['filter_min_avg_profit'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
         epochs = [
             x for x in epochs
-            if x['results_metrics']['avg_profit'] > filteroptions['filter_min_avg_profit']
+            if x['results_metrics'].get(
+                'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
+            ) > filteroptions['filter_min_avg_profit']
         ]
     if filteroptions['filter_max_avg_profit'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
         epochs = [
             x for x in epochs
-            if x['results_metrics']['avg_profit'] < filteroptions['filter_max_avg_profit']
+            if x['results_metrics'].get(
+                'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
+                ) < filteroptions['filter_max_avg_profit']
         ]
     if filteroptions['filter_min_total_profit'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
         epochs = [
             x for x in epochs
-            if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit']
+            if x['results_metrics'].get(
+                'profit', x['results_metrics'].get('profit_total_abs', 0)
+                ) > filteroptions['filter_min_total_profit']
         ]
     if filteroptions['filter_max_total_profit'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
         epochs = [
             x for x in epochs
-            if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit']
+            if x['results_metrics'].get(
+                'profit', x['results_metrics'].get('profit_total_abs', 0)
+                ) < filteroptions['filter_max_total_profit']
         ]
     return epochs
 
@@ -218,11 +256,11 @@ def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
 def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
 
     if filteroptions['filter_min_objective'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
 
         epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
     if filteroptions['filter_max_objective'] is not None:
-        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+        epochs = _hyperopt_filter_epochs_trade(epochs, 0)
 
         epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
 

freqtrade/configuration/configuration.py

@@ -75,8 +75,6 @@ class Configuration:
         # Normalize config
         if 'internals' not in config:
             config['internals'] = {}
-        # TODO: This can be deleted along with removal of deprecated
-        # experimental settings
         if 'ask_strategy' not in config:
             config['ask_strategy'] = {}
 
@@ -108,6 +106,8 @@ class Configuration:
 
         self._process_plot_options(config)
 
+        self._process_data_options(config)
+
         # Check if the exchange set by the user is supported
         check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))
 
@@ -399,6 +399,11 @@ class Configuration:
         self._args_to_config(config, argname='dataformat_trades',
                              logstring='Using "{}" to store trades data.')
 
+    def _process_data_options(self, config: Dict[str, Any]) -> None:
+
+        self._args_to_config(config, argname='new_pairs_days',
+                             logstring='Detected --new-pairs-days: {}')
+
     def _process_runmode(self, config: Dict[str, Any]) -> None:
 
         self._args_to_config(config, argname='dry_run',
@@ -445,6 +450,7 @@ class Configuration:
         """
 
         if "pairs" in config:
+            config['exchange']['pair_whitelist'] = config['pairs']
             return
 
         if "pairs_file" in self.args and self.args["pairs_file"]:

freqtrade/configuration/timerange.py

@@ -3,6 +3,7 @@ This module contains the argument manager class
 """
 import logging
 import re
+from datetime import datetime
 from typing import Optional
 
 import arrow
@@ -43,7 +44,7 @@ class TimeRange:
             self.startts = self.startts - seconds
 
     def adjust_start_if_necessary(self, timeframe_secs: int, startup_candles: int,
-                                  min_date: arrow.Arrow) -> None:
+                                  min_date: datetime) -> None:
         """
         Adjust startts by <startup_candles> candles.
         Applies only if no startup-candles have been available.
@@ -54,11 +55,11 @@ class TimeRange:
         :return: None (Modifies the object in place)
         """
         if (not self.starttype or (startup_candles
-                                   and min_date.int_timestamp >= self.startts)):
+                                   and min_date.timestamp() >= self.startts)):
             # If no startts was defined, or backtest-data starts at the defined backtest-date
             logger.warning("Moving start-date by %s candles to account for startup time.",
                            startup_candles)
-            self.startts = (min_date.int_timestamp + timeframe_secs * startup_candles)
+            self.startts = int(min_date.timestamp() + timeframe_secs * startup_candles)
             self.starttype = 'date'
 
     @staticmethod

freqtrade/constants.py

@@ -11,6 +11,7 @@ DEFAULT_EXCHANGE = 'bittrex'
 PROCESS_THROTTLE_SECS = 5  # sec
 HYPEROPT_EPOCH = 100  # epochs
 RETRY_TIMEOUT = 30  # sec
+TIMEOUT_UNITS = ['minutes', 'seconds']
 DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite'
 DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite'
 UNLIMITED_STAKE_AMOUNT = 'unlimited'
@@ -96,6 +97,7 @@ CONF_SCHEMA = {
     'type': 'object',
     'properties': {
         'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1},
+        'new_pairs_days': {'type': 'integer', 'default': 30},
         'timeframe': {'type': 'string'},
         'stake_currency': {'type': 'string'},
         'stake_amount': {
@@ -136,7 +138,8 @@ CONF_SCHEMA = {
             'type': 'object',
             'properties': {
                 'buy': {'type': 'number', 'minimum': 1},
-                'sell': {'type': 'number', 'minimum': 1}
+                'sell': {'type': 'number', 'minimum': 1},
+                'unit': {'type': 'string', 'enum': TIMEOUT_UNITS, 'default': 'minutes'}
             }
         },
         'bid_strategy': {
@@ -246,14 +249,24 @@ CONF_SCHEMA = {
                 'balance_dust_level': {'type': 'number', 'minimum': 0.0},
                 'notification_settings': {
                     'type': 'object',
+                    'default': {},
                     'properties': {
                         'status': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
                         'warning': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
                         'startup': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
                         'buy': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
-                        'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
                         'buy_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
-                        'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}
+                        'buy_fill': {'type': 'string',
+                                     'enum': TELEGRAM_SETTING_OPTIONS,
+                                     'default': 'off'
+                                     },
+                        'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
+                        'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
+                        'sell_fill': {
+                            'type': 'string',
+                            'enum': TELEGRAM_SETTING_OPTIONS,
+                            'default': 'off'
+                            },
                     }
                 }
             },

freqtrade/data/btanalysis.py

@@ -156,6 +156,7 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
 
         data = data['strategy'][strategy]['trades']
         df = pd.DataFrame(data)
+        if not df.empty:
             df['open_date'] = pd.to_datetime(df['open_date'],
                                              utc=True,
                                              infer_datetime_format=True
@@ -167,7 +168,7 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
     else:
         # old format - only with lists.
         df = pd.DataFrame(data, columns=BT_DATA_COLUMNS_OLD)
-
+        if not df.empty:
             df['open_date'] = pd.to_datetime(df['open_date'],
                                              unit='s',
                                              utc=True,
@@ -180,6 +181,7 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
                                               )
             # Create compatibility with new format
             df['profit_abs'] = df['close_rate'] - df['open_rate']
+    if not df.empty:
         if 'profit_ratio' not in df.columns:
             df['profit_ratio'] = df['profit_percent']
         df = df.sort_values("open_date").reset_index(drop=True)
@@ -337,7 +339,7 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
     """
     Adds a column `col_name` with the cumulative profit for the given trades array.
     :param df: DataFrame with date index
-    :param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
+    :param trades: DataFrame containing trades (requires columns close_date and profit_abs)
     :param col_name: Column name that will be assigned the results
     :param timeframe: Timeframe used during the operations
     :return: Returns df with one additional column, col_name, containing the cumulative profit.
@@ -349,8 +351,8 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
     timeframe_minutes = timeframe_to_minutes(timeframe)
     # Resample to timeframe to make sure trades match candles
     _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date'
-                                  )[['profit_ratio']].sum()
+                                  )[['profit_abs']].sum()
-    df.loc[:, col_name] = _trades_sum['profit_ratio'].cumsum()
+    df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum()
     # Set first value to 0
     df.loc[df.iloc[0].name, col_name] = 0
     # FFill to get continuous

freqtrade/data/converter.py

@@ -145,6 +145,27 @@ def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date',
     return df
 
 
+def trim_dataframes(preprocessed: Dict[str, DataFrame], timerange,
+                    startup_candles: int) -> Dict[str, DataFrame]:
+    """
+    Trim startup period from analyzed dataframes
+    :param preprocessed: Dict of pair: dataframe
+    :param timerange: timerange (use start and end date if available)
+    :param startup_candles: Startup-candles that should be removed
+    :return: Dict of trimmed dataframes
+    """
+    processed: Dict[str, DataFrame] = {}
+
+    for pair, df in preprocessed.items():
+        trimed_df = trim_dataframe(df, timerange, startup_candles=startup_candles)
+        if not trimed_df.empty:
+            processed[pair] = trimed_df
+        else:
+            logger.warning(f'{pair} has no data left after adjusting for startup candles, '
+                           f'skipping.')
+    return processed
+
+
 def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
     """
     TODO: This should get a dedicated test

freqtrade/data/dataprovider.py

@@ -19,14 +19,25 @@ from freqtrade.state import RunMode
 
 logger = logging.getLogger(__name__)
 
+NO_EXCHANGE_EXCEPTION = 'Exchange is not available to DataProvider.'
+MAX_DATAFRAME_CANDLES = 1000
+
 
 class DataProvider:
 
-    def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None:
+    def __init__(self, config: dict, exchange: Optional[Exchange], pairlists=None) -> None:
         self._config = config
         self._exchange = exchange
         self._pairlists = pairlists
         self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
+        self.__slice_index: Optional[int] = None
+
+    def _set_dataframe_max_index(self, limit_index: int):
+        """
+        Limit analyzed dataframe to max specified index.
+        :param limit_index: dataframe index.
+        """
+        self.__slice_index = limit_index
 
     def _set_cached_df(self, pair: str, timeframe: str, dataframe: DataFrame) -> None:
         """
@@ -45,40 +56,6 @@ class DataProvider:
         """
         self._pairlists = pairlists
 
-    def refresh(self,
-                pairlist: ListPairsWithTimeframes,
-                helping_pairs: ListPairsWithTimeframes = None) -> None:
-        """
-        Refresh data, called with each cycle
-        """
-        if helping_pairs:
-            self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs)
-        else:
-            self._exchange.refresh_latest_ohlcv(pairlist)
-
-    @property
-    def available_pairs(self) -> ListPairsWithTimeframes:
-        """
-        Return a list of tuples containing (pair, timeframe) for which data is currently cached.
-        Should be whitelist + open trades.
-        """
-        return list(self._exchange._klines.keys())
-
-    def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame:
-        """
-        Get candle (OHLCV) data for the given pair as DataFrame
-        Please use the `available_pairs` method to verify which pairs are currently cached.
-        :param pair: pair to get the data for
-        :param timeframe: Timeframe to get data for
-        :param copy: copy dataframe before returning if True.
-                     Use False only for read-only operations (where the dataframe is not modified)
-        """
-        if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
-            return self._exchange.klines((pair, timeframe or self._config['timeframe']),
-                                         copy=copy)
-        else:
-            return DataFrame()
-
     def historic_ohlcv(self, pair: str, timeframe: str = None) -> DataFrame:
         """
         Get stored historical candle (OHLCV) data
@@ -111,47 +88,27 @@ class DataProvider:
 
     def get_analyzed_dataframe(self, pair: str, timeframe: str) -> Tuple[DataFrame, datetime]:
         """
+        Retrieve the analyzed dataframe. Returns the full dataframe in trade mode (live / dry),
+        and the last 1000 candles (up to the time evaluated at this moment) in all other modes.
         :param pair: pair to get the data for
         :param timeframe: timeframe to get data for
         :return: Tuple of (Analyzed Dataframe, lastrefreshed) for the requested pair / timeframe
             combination.
             Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached.
         """
-        if (pair, timeframe) in self.__cached_pairs:
+        pair_key = (pair, timeframe)
-            return self.__cached_pairs[(pair, timeframe)]
+        if pair_key in self.__cached_pairs:
+            if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
+                df, date = self.__cached_pairs[pair_key]
+            else:
+                df, date = self.__cached_pairs[pair_key]
+                if self.__slice_index is not None:
+                    max_index = self.__slice_index
+                    df = df.iloc[max(0, max_index - MAX_DATAFRAME_CANDLES):max_index]
+            return df, date
         else:
-
             return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc))
 
-    def market(self, pair: str) -> Optional[Dict[str, Any]]:
-        """
-        Return market data for the pair
-        :param pair: Pair to get the data for
-        :return: Market data dict from ccxt or None if market info is not available for the pair
-        """
-        return self._exchange.markets.get(pair)
-
-    def ticker(self, pair: str):
-        """
-        Return last ticker data from exchange
-        :param pair: Pair to get the data for
-        :return: Ticker dict from exchange or empty dict if ticker is not available for the pair
-        """
-        try:
-            return self._exchange.fetch_ticker(pair)
-        except ExchangeError:
-            return {}
-
-    def orderbook(self, pair: str, maximum: int) -> Dict[str, List]:
-        """
-        Fetch latest l2 orderbook data
-        Warning: Does a network request - so use with common sense.
-        :param pair: pair to get the data for
-        :param maximum: Maximum number of orderbook entries to query
-        :return: dict including bids/asks with a total of `maximum` entries.
-        """
-        return self._exchange.fetch_l2_order_book(pair, maximum)
-
     @property
     def runmode(self) -> RunMode:
         """
@@ -170,6 +127,89 @@ class DataProvider:
         """
 
         if self._pairlists:
-            return self._pairlists.whitelist
+            return self._pairlists.whitelist.copy()
         else:
             raise OperationalException("Dataprovider was not initialized with a pairlist provider.")
+
+    def clear_cache(self):
+        """
+        Clear pair dataframe cache.
+        """
+        self.__cached_pairs = {}
+
+    # Exchange functions
+
+    def refresh(self,
+                pairlist: ListPairsWithTimeframes,
+                helping_pairs: ListPairsWithTimeframes = None) -> None:
+        """
+        Refresh data, called with each cycle
+        """
+        if self._exchange is None:
+            raise OperationalException(NO_EXCHANGE_EXCEPTION)
+        if helping_pairs:
+            self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs)
+        else:
+            self._exchange.refresh_latest_ohlcv(pairlist)
+
+    @property
+    def available_pairs(self) -> ListPairsWithTimeframes:
+        """
+        Return a list of tuples containing (pair, timeframe) for which data is currently cached.
+        Should be whitelist + open trades.
+        """
+        if self._exchange is None:
+            raise OperationalException(NO_EXCHANGE_EXCEPTION)
+        return list(self._exchange._klines.keys())
+
+    def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame:
+        """
+        Get candle (OHLCV) data for the given pair as DataFrame
+        Please use the `available_pairs` method to verify which pairs are currently cached.
+        :param pair: pair to get the data for
+        :param timeframe: Timeframe to get data for
+        :param copy: copy dataframe before returning if True.
+                     Use False only for read-only operations (where the dataframe is not modified)
+        """
+        if self._exchange is None:
+            raise OperationalException(NO_EXCHANGE_EXCEPTION)
+        if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
+            return self._exchange.klines((pair, timeframe or self._config['timeframe']),
+                                         copy=copy)
+        else:
+            return DataFrame()
+
+    def market(self, pair: str) -> Optional[Dict[str, Any]]:
+        """
+        Return market data for the pair
+        :param pair: Pair to get the data for
+        :return: Market data dict from ccxt or None if market info is not available for the pair
+        """
+        if self._exchange is None:
+            raise OperationalException(NO_EXCHANGE_EXCEPTION)
+        return self._exchange.markets.get(pair)
+
+    def ticker(self, pair: str):
+        """
+        Return last ticker data from exchange
+        :param pair: Pair to get the data for
+        :return: Ticker dict from exchange or empty dict if ticker is not available for the pair
+        """
+        if self._exchange is None:
+            raise OperationalException(NO_EXCHANGE_EXCEPTION)
+        try:
+            return self._exchange.fetch_ticker(pair)
+        except ExchangeError:
+            return {}
+
+    def orderbook(self, pair: str, maximum: int) -> Dict[str, List]:
+        """
+        Fetch latest l2 orderbook data
+        Warning: Does a network request - so use with common sense.
+        :param pair: pair to get the data for
+        :param maximum: Maximum number of orderbook entries to query
+        :return: dict including bids/asks with a total of `maximum` entries.
+        """
+        if self._exchange is None:
+            raise OperationalException(NO_EXCHANGE_EXCEPTION)
+        return self._exchange.fetch_l2_order_book(pair, maximum)

freqtrade/data/history/hdf5datahandler.py

@@ -89,7 +89,7 @@ class HDF5DataHandler(IDataHandler):
             if timerange.starttype == 'date':
                 where.append(f"date >= Timestamp({timerange.startts * 1e9})")
             if timerange.stoptype == 'date':
-                where.append(f"date < Timestamp({timerange.stopts * 1e9})")
+                where.append(f"date <= Timestamp({timerange.stopts * 1e9})")
 
         pairdata = pd.read_hdf(filename, key=key, mode="r", where=where)
 

freqtrade/data/history/history_utils.py

@@ -155,6 +155,7 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
 def _download_pair_history(datadir: Path,
                            exchange: Exchange,
                            pair: str, *,
+                           new_pairs_days: int = 30,
                            timeframe: str = '5m',
                            timerange: Optional[TimeRange] = None,
                            data_handler: IDataHandler = None) -> bool:
@@ -193,7 +194,7 @@ def _download_pair_history(datadir: Path,
                                                timeframe=timeframe,
                                                since_ms=since_ms if since_ms else
                                                int(arrow.utcnow().shift(
-                                                   days=-30).float_timestamp) * 1000
+                                                   days=-new_pairs_days).float_timestamp) * 1000
                                                )
         # TODO: Maybe move parsing to exchange class (?)
         new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
@@ -223,7 +224,8 @@ def _download_pair_history(datadir: Path,
 
 def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
                                 datadir: Path, timerange: Optional[TimeRange] = None,
-                                erase: bool = False, data_format: str = None) -> List[str]:
+                                new_pairs_days: int = 30, erase: bool = False,
+                                data_format: str = None) -> List[str]:
     """
     Refresh stored ohlcv data for backtesting and hyperopt operations.
     Used by freqtrade download-data subcommand.
@@ -246,12 +248,14 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
             logger.info(f'Downloading pair {pair}, interval {timeframe}.')
             _download_pair_history(datadir=datadir, exchange=exchange,
                                    pair=pair, timeframe=str(timeframe),
+                                   new_pairs_days=new_pairs_days,
                                    timerange=timerange, data_handler=data_handler)
     return pairs_not_available
 
 
 def _download_trades_history(exchange: Exchange,
                              pair: str, *,
+                             new_pairs_days: int = 30,
                              timerange: Optional[TimeRange] = None,
                              data_handler: IDataHandler
                              ) -> bool:
@@ -261,9 +265,13 @@ def _download_trades_history(exchange: Exchange,
     """
     try:
 
-        since = timerange.startts * 1000 if \
+        until = None
-            (timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift(
+        if (timerange and timerange.starttype == 'date'):
-                days=-30).float_timestamp) * 1000
+            since = timerange.startts * 1000
+            if timerange.stoptype == 'date':
+                until = timerange.stopts * 1000
+        else:
+            since = int(arrow.utcnow().shift(days=-new_pairs_days).float_timestamp) * 1000
 
         trades = data_handler.trades_load(pair)
 
@@ -291,6 +299,7 @@ def _download_trades_history(exchange: Exchange,
         # Default since_ms to 30 days if nothing is given
         new_trades = exchange.get_historic_trades(pair=pair,
                                                   since=since,
+                                                  until=until,
                                                   from_id=from_id,
                                                   )
         trades.extend(new_trades[1])
@@ -311,8 +320,8 @@ def _download_trades_history(exchange: Exchange,
 
 
 def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
-                                 timerange: TimeRange, erase: bool = False,
+                                 timerange: TimeRange, new_pairs_days: int = 30,
-                                 data_format: str = 'jsongz') -> List[str]:
+                                 erase: bool = False, data_format: str = 'jsongz') -> List[str]:
     """
     Refresh stored trades data for backtesting and hyperopt operations.
     Used by freqtrade download-data subcommand.
@@ -333,6 +342,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
         logger.info(f'Downloading trades for pair {pair}.')
         _download_trades_history(exchange=exchange,
                                  pair=pair,
+                                 new_pairs_days=new_pairs_days,
                                  timerange=timerange,
                                  data_handler=data_handler)
     return pairs_not_available
@@ -362,7 +372,7 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
                 logger.exception(f'Could not convert {pair} to OHLCV.')
 
 
-def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
+def get_timerange(data: Dict[str, DataFrame]) -> Tuple[datetime, datetime]:
     """
     Get the maximum common timerange for the given backtest data.
 
@@ -370,7 +380,7 @@ def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]
     :return: tuple containing min_date, max_date
     """
     timeranges = [
-        (arrow.get(frame['date'].min()), arrow.get(frame['date'].max()))
+        (frame['date'].min().to_pydatetime(), frame['date'].max().to_pydatetime())
         for frame in data.values()
     ]
     return (min(timeranges, key=operator.itemgetter(0))[0],

freqtrade/edge/edge_positioning.py

@@ -1,6 +1,8 @@
 # pragma pylint: disable=W0603
 """ Edge positioning package """
 import logging
+from collections import defaultdict
+from copy import deepcopy
 from typing import Any, Dict, List, NamedTuple
 
 import arrow
@@ -12,8 +14,10 @@ from freqtrade.configuration import TimeRange
 from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT
 from freqtrade.data.history import get_timerange, load_data, refresh_data
 from freqtrade.exceptions import OperationalException
+from freqtrade.exchange.exchange import timeframe_to_seconds
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
-from freqtrade.strategy.interface import SellType
+from freqtrade.state import RunMode
+from freqtrade.strategy.interface import IStrategy, SellType
 
 
 logger = logging.getLogger(__name__)
@@ -45,7 +49,7 @@ class Edge:
 
         self.config = config
         self.exchange = exchange
-        self.strategy = strategy
+        self.strategy: IStrategy = strategy
 
         self.edge_config = self.config.get('edge', {})
         self._cached_pairs: Dict[str, Any] = {}  # Keeps a list of pairs
@@ -81,10 +85,15 @@ class Edge:
         if config.get('fee'):
             self.fee = config['fee']
         else:
+            try:
                 self.fee = self.exchange.get_fee(symbol=expand_pairlist(
                     self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0])
+            except IndexError:
+                self.fee = None
 
     def calculate(self, pairs: List[str]) -> bool:
+        if self.fee is None and pairs:
+            self.fee = self.exchange.get_fee(pairs[0])
 
         heartbeat = self.edge_config.get('process_throttle_secs')
 
@@ -97,12 +106,31 @@ class Edge:
         logger.info('Using local backtesting data (using whitelist in given config) ...')
 
         if self._refresh_pairs:
+            timerange_startup = deepcopy(self._timerange)
+            timerange_startup.subtract_start(timeframe_to_seconds(
+                self.strategy.timeframe) * self.strategy.startup_candle_count)
             refresh_data(
                 datadir=self.config['datadir'],
                 pairs=pairs,
                 exchange=self.exchange,
                 timeframe=self.strategy.timeframe,
-                timerange=self._timerange,
+                timerange=timerange_startup,
+                data_format=self.config.get('dataformat_ohlcv', 'json'),
+            )
+            # Download informative pairs too
+            res = defaultdict(list)
+            for p, t in self.strategy.informative_pairs():
+                res[t].append(p)
+            for timeframe, inf_pairs in res.items():
+                timerange_startup = deepcopy(self._timerange)
+                timerange_startup.subtract_start(timeframe_to_seconds(
+                    timeframe) * self.strategy.startup_candle_count)
+                refresh_data(
+                    datadir=self.config['datadir'],
+                    pairs=inf_pairs,
+                    exchange=self.exchange,
+                    timeframe=timeframe,
+                    timerange=timerange_startup,
                     data_format=self.config.get('dataformat_ohlcv', 'json'),
                 )
 
@@ -120,8 +148,11 @@ class Edge:
             self._cached_pairs = {}
             logger.critical("No data found. Edge is stopped ...")
             return False
-
+        # Fake run-mode to Edge
+        prior_rm = self.config['runmode']
+        self.config['runmode'] = RunMode.EDGE
         preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
+        self.config['runmode'] = prior_rm
 
         # Print timeframe
         min_date, max_date = get_timerange(preprocessed)
@@ -178,7 +209,7 @@ class Edge:
         if pair in self._cached_pairs:
             return self._cached_pairs[pair].stoploss
         else:
-            logger.warning('tried to access stoploss of a non-existing pair, '
+            logger.warning(f'Tried to access stoploss of non-existing pair {pair}, '
                            'strategy stoploss is returned instead.')
             return self.strategy.stoploss
 
@@ -209,7 +240,7 @@ class Edge:
 
         return self._final_pairs
 
-    def accepted_pairs(self) -> list:
+    def accepted_pairs(self) -> List[Dict[str, Any]]:
         """
         return a list of accepted pairs along with their winrate, expectancy and stoploss
         """

freqtrade/exchange/__init__.py

@@ -15,4 +15,6 @@ from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges,
                                          timeframe_to_seconds, validate_exchange,
                                          validate_exchanges)
 from freqtrade.exchange.ftx import Ftx
+from freqtrade.exchange.hitbtc import Hitbtc
 from freqtrade.exchange.kraken import Kraken
+from freqtrade.exchange.kucoin import Kucoin

freqtrade/exchange/bittrex.py

@@ -12,10 +12,6 @@ class Bittrex(Exchange):
     """
     Bittrex exchange class. Contains adjustments needed for Freqtrade to work
     with this exchange.
-
-    Please note that this exchange is not included in the list of exchanges
-    officially supported by the Freqtrade development team. So some features
-    may still not work as expected.
     """
 
     _ft_has: Dict = {

freqtrade/exchange/exchange.py

@@ -14,6 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple
 import arrow
 import ccxt
 import ccxt.async_support as ccxt_async
+from cachetools import TTLCache
 from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE,
                                             decimal_to_precision)
 from pandas import DataFrame
@@ -58,11 +59,13 @@ class Exchange:
     _ft_has_default: Dict = {
         "stoploss_on_exchange": False,
         "order_time_in_force": ["gtc"],
+        "ohlcv_params": {},
         "ohlcv_candle_limit": 500,
         "ohlcv_partial_candle": True,
         "trades_pagination": "time",  # Possible are "time" or "id"
         "trades_pagination_arg": "since",
         "l2_limit_range": None,
+        "l2_limit_range_required": True,  # Allow Empty L2 limit (kucoin)
     }
     _ft_has: Dict = {}
 
@@ -83,6 +86,9 @@ class Exchange:
         # Timestamp of last markets refresh
         self._last_markets_refresh: int = 0
 
+        # Cache for 10 minutes ...
+        self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=1, ttl=60 * 10)
+
         # Holds candles
         self._klines: Dict[Tuple[str, str], DataFrame] = {}
 
@@ -358,7 +364,6 @@ class Exchange:
         invalid_pairs = []
         for pair in extended_pairs:
             # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs
-            # TODO: add a support for having coins in BTC/USDT format
             if self.markets and pair not in self.markets:
                 raise OperationalException(
                     f'Pair {pair} is not available on {self.name}. '
@@ -461,7 +466,7 @@ class Exchange:
     def amount_to_precision(self, pair: str, amount: float) -> float:
         '''
         Returns the amount to buy or sell to a precision the Exchange accepts
-        Reimplementation of ccxt internal methods - ensuring we can test the result is correct
+        Re-implementation of ccxt internal methods - ensuring we can test the result is correct
         based on our definitions.
         '''
         if self.markets[pair]['precision']['amount']:
@@ -475,7 +480,7 @@ class Exchange:
     def price_to_precision(self, pair: str, price: float) -> float:
         '''
         Returns the price rounded up to the precision the Exchange accepts.
-        Partial Reimplementation of ccxt internal method decimal_to_precision(),
+        Partial Re-implementation of ccxt internal method decimal_to_precision(),
         which does not support rounding up
         TODO: If ccxt supports ROUND_UP for decimal_to_precision(), we could remove this and
         align with amount_to_precision().
@@ -534,7 +539,9 @@ class Exchange:
         # reserve some percent defined in config (5% default) + stoploss
         amount_reserve_percent = 1.0 + self._config.get('amount_reserve_percent',
                                                         DEFAULT_AMOUNT_RESERVE_PERCENT)
-        amount_reserve_percent += abs(stoploss)
+        amount_reserve_percent = (
+          amount_reserve_percent / (1 - abs(stoploss)) if abs(stoploss) != 1 else 1.5
+        )
         # it should not be more than 50%
         amount_reserve_percent = max(min(amount_reserve_percent, 1.5), 1)
 
@@ -660,17 +667,6 @@ class Exchange:
 
         raise OperationalException(f"stoploss is not implemented for {self.name}.")
 
-    @retrier
-    def get_balance(self, currency: str) -> float:
-
-        # ccxt exception is already handled by get_balances
-        balances = self.get_balances()
-        balance = balances.get(currency)
-        if balance is None:
-            raise TemporaryError(
-                f'Could not get {currency} balance due to malformed exchange response: {balances}')
-        return balance['free']
-
     @retrier
     def get_balances(self) -> dict:
 
@@ -692,9 +688,19 @@ class Exchange:
             raise OperationalException(e) from e
 
     @retrier
-    def get_tickers(self) -> Dict:
+    def get_tickers(self, cached: bool = False) -> Dict:
+        """
+        :param cached: Allow cached result
+        :return: fetch_tickers result
+        """
+        if cached:
+            tickers = self._fetch_tickers_cache.get('fetch_tickers')
+            if tickers:
+                return tickers
         try:
-            return self._api.fetch_tickers()
+            tickers = self._api.fetch_tickers()
+            self._fetch_tickers_cache['fetch_tickers'] = tickers
+            return tickers
         except ccxt.NotSupported as e:
             raise OperationalException(
                 f'Exchange {self._api.name} does not support fetching tickers in batch. '
@@ -857,10 +863,11 @@ class Exchange:
                 "Fetching pair %s, interval %s, since %s %s...",
                 pair, timeframe, since_ms, s
             )
-
+            params = self._ft_has.get('ohlcv_params', {})
             data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe,
                                                      since=since_ms,
-                                                     limit=self.ohlcv_candle_limit(timeframe))
+                                                     limit=self.ohlcv_candle_limit(timeframe),
+                                                     params=params)
 
             # Some exchanges sort OHLCV in ASC order and others in DESC.
             # Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last)
@@ -1113,6 +1120,27 @@ class Exchange:
 
         return order
 
+    def cancel_stoploss_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict:
+        """
+        Cancel stoploss order returning a result.
+        Creates a fake result if cancel order returns a non-usable result
+        and fetch_order does not work (certain exchanges don't return cancelled orders)
+        :param order_id: stoploss-order-id to cancel
+        :param pair: Pair corresponding to order_id
+        :param amount: Amount to use for fake response
+        :return: Result from either cancel_order if usable, or fetch_order
+        """
+        corder = self.cancel_stoploss_order(order_id, pair)
+        if self.is_cancel_order_result_suitable(corder):
+            return corder
+        try:
+            order = self.fetch_stoploss_order(order_id, pair)
+        except InvalidOrderException:
+            logger.warning(f"Could not fetch cancelled stoploss order {order_id}.")
+            order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
+
+        return order
+
     @retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
     def fetch_order(self, order_id: str, pair: str) -> Dict:
         if self._config['dry_run']:
@@ -1154,14 +1182,20 @@ class Exchange:
         return self.fetch_order(order_id, pair)
 
     @staticmethod
-    def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]]):
+    def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]],
+                               range_required: bool = True):
         """
         Get next greater value in the list.
         Used by fetch_l2_order_book if the api only supports a limited range
         """
         if not limit_range:
             return limit
-        return min([x for x in limit_range if limit <= x] + [max(limit_range)])
+
+        result = min([x for x in limit_range if limit <= x] + [max(limit_range)])
+        if not range_required and limit > result:
+            # Range is not required - we can use None as parameter.
+            return None
+        return result
 
     @retrier
     def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
@@ -1171,7 +1205,8 @@ class Exchange:
         Returns a dict in the format
         {'asks': [price, volume], 'bids': [price, volume]}
         """
-        limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'])
+        limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'],
+                                             self._ft_has['l2_limit_range_required'])
         try:
 
             return self._api.fetch_l2_order_book(pair, limit1)
@@ -1225,6 +1260,9 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
+    def get_order_id_conditional(self, order: Dict[str, Any]) -> str:
+        return order['id']
+
     @retrier
     def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1,
                 price: float = 1, taker_or_maker: str = 'maker') -> float:

freqtrade/exchange/ftx.py

@@ -8,6 +8,7 @@ from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, Invali
                                   OperationalException, TemporaryError)
 from freqtrade.exchange import Exchange
 from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT, retrier
+from freqtrade.misc import safe_value_fallback2
 
 
 logger = logging.getLogger(__name__)
@@ -63,10 +64,11 @@ class Ftx(Exchange):
                 # set orderPrice to place limit order, otherwise it's a market order
                 params['orderPrice'] = limit_rate
 
+            params['stopPrice'] = stop_price
             amount = self.amount_to_precision(pair, amount)
 
             order = self._api.create_order(symbol=pair, type=ordertype, side='sell',
-                                           amount=amount, price=stop_price, params=params)
+                                           amount=amount, params=params)
             logger.info('stoploss order added for %s. '
                         'stop price: %s.', pair, stop_price)
             return order
@@ -134,3 +136,8 @@ class Ftx(Exchange):
                 f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
+
+    def get_order_id_conditional(self, order: Dict[str, Any]) -> str:
+        if order['type'] == 'stop':
+            return safe_value_fallback2(order['info'], order, 'orderId', 'id')
+        return order['id']

freqtrade/exchange/hitbtc.py

@@ -0,0 +1,24 @@
+import logging
+from typing import Dict
+
+from freqtrade.exchange import Exchange
+
+
+logger = logging.getLogger(__name__)
+
+
+class Hitbtc(Exchange):
+    """
+    Hitbtc exchange class. Contains adjustments needed for Freqtrade to work
+    with this exchange.
+
+    Please note that this exchange is not included in the list of exchanges
+    officially supported by the Freqtrade development team. So some features
+    may still not work as expected.
+    """
+
+    # fetchCurrencies API point requires authentication for Hitbtc,
+    _ft_has: Dict = {
+        "ohlcv_candle_limit": 1000,
+        "ohlcv_params": {"sort": "DESC"}
+    }

freqtrade/exchange/kraken.py

@@ -53,6 +53,8 @@ class Kraken(Exchange):
                            # x["side"], x["amount"],
                            ) for x in orders]
             for bal in balances:
+                if not isinstance(balances[bal], dict):
+                    continue
                 balances[bal]['used'] = sum(order[1] for order in order_list if order[0] == bal)
                 balances[bal]['free'] = balances[bal]['total'] - balances[bal]['used']
 

freqtrade/exchange/kucoin.py

@@ -0,0 +1,24 @@
+""" Kucoin exchange subclass """
+import logging
+from typing import Dict
+
+from freqtrade.exchange import Exchange
+
+
+logger = logging.getLogger(__name__)
+
+
+class Kucoin(Exchange):
+    """
+    Kucoin exchange class. Contains adjustments needed for Freqtrade to work
+    with this exchange.
+
+    Please note that this exchange is not included in the list of exchanges
+    officially supported by the Freqtrade development team. So some features
+    may still not work as expected.
+    """
+
+    _ft_has: Dict = {
+        "l2_limit_range": [20, 100],
+        "l2_limit_range_required": False,
+    }

freqtrade/freqtradebot.py

@@ -28,7 +28,7 @@ from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.rpc import RPCManager, RPCMessageType
 from freqtrade.state import State
-from freqtrade.strategy.interface import IStrategy, SellType
+from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
 from freqtrade.wallets import Wallets
 
@@ -113,7 +113,7 @@ class FreqtradeBot(LoggingMixin):
         via RPC about changes in the bot status.
         """
         self.rpc.send_msg({
-            'type': RPCMessageType.STATUS_NOTIFICATION,
+            'type': RPCMessageType.STATUS,
             'status': msg
         })
 
@@ -205,7 +205,7 @@ class FreqtradeBot(LoggingMixin):
 
         if len(open_trades) != 0:
             msg = {
-                'type': RPCMessageType.WARNING_NOTIFICATION,
+                'type': RPCMessageType.WARNING,
                 'status':  f"{len(open_trades)} open trades active.\n\n"
                            f"Handle these trades manually on {self.exchange.name}, "
                            f"or '/start' the bot again and use '/stopbuy' "
@@ -267,7 +267,7 @@ class FreqtradeBot(LoggingMixin):
     def update_closed_trades_without_assigned_fees(self):
         """
         Update closed trades without close fees assigned.
-        Only acts when Orders are in the database, otherwise the last orderid is unknown.
+        Only acts when Orders are in the database, otherwise the last order-id is unknown.
         """
         if self.config['dry_run']:
             # Updating open orders in dry-run does not make sense and will fail.
@@ -378,7 +378,7 @@ class FreqtradeBot(LoggingMixin):
             if lock:
                 self.log_once(f"Global pairlock active until "
                               f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)}. "
-                              "Not creating new trades.", logger.info)
+                              f"Not creating new trades, reason: {lock.reason}.", logger.info)
             else:
                 self.log_once("Global pairlock active. Not creating new trades.", logger.info)
             return trades_created
@@ -456,7 +456,8 @@ class FreqtradeBot(LoggingMixin):
             lock = PairLocks.get_pair_longest_lock(pair, nowtime)
             if lock:
                 self.log_once(f"Pair {pair} is still locked until "
-                              f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)}.",
+                              f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)} "
+                              f"due to {lock.reason}.",
                               logger.info)
             else:
                 self.log_once(f"Pair {pair} is still locked.", logger.info)
@@ -472,8 +473,7 @@ class FreqtradeBot(LoggingMixin):
         (buy, sell) = self.strategy.get_signal(pair, self.strategy.timeframe, analyzed_df)
 
         if buy and not sell:
-            stake_amount = self.wallets.get_trade_stake_amount(pair, self.get_free_open_trades(),
+            stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge)
-                                                               self.edge)
             if not stake_amount:
                 logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.")
                 return False
@@ -552,7 +552,7 @@ class FreqtradeBot(LoggingMixin):
 
         if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)(
                 pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested,
-                time_in_force=time_in_force):
+                time_in_force=time_in_force, current_time=datetime.now(timezone.utc)):
             logger.info(f"User requested abortion of buying {pair}")
             return False
         amount = self.exchange.amount_to_precision(pair, amount)
@@ -601,6 +601,7 @@ class FreqtradeBot(LoggingMixin):
             pair=pair,
             stake_amount=stake_amount,
             amount=amount,
+            is_open=True,
             amount_requested=amount_requested,
             fee_open=fee,
             fee_close=fee,
@@ -630,11 +631,11 @@ class FreqtradeBot(LoggingMixin):
 
     def _notify_buy(self, trade: Trade, order_type: str) -> None:
         """
-        Sends rpc notification when a buy occured.
+        Sends rpc notification when a buy occurred.
         """
         msg = {
             'trade_id': trade.id,
-            'type': RPCMessageType.BUY_NOTIFICATION,
+            'type': RPCMessageType.BUY,
             'exchange': self.exchange.name.capitalize(),
             'pair': trade.pair,
             'limit': trade.open_rate,
@@ -652,13 +653,13 @@ class FreqtradeBot(LoggingMixin):
 
     def _notify_buy_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
         """
-        Sends rpc notification when a buy cancel occured.
+        Sends rpc notification when a buy cancel occurred.
         """
         current_rate = self.get_buy_rate(trade.pair, False)
 
         msg = {
             'trade_id': trade.id,
-            'type': RPCMessageType.BUY_CANCEL_NOTIFICATION,
+            'type': RPCMessageType.BUY_CANCEL,
             'exchange': self.exchange.name.capitalize(),
             'pair': trade.pair,
             'limit': trade.open_rate,
@@ -675,6 +676,21 @@ class FreqtradeBot(LoggingMixin):
         # Send the message
         self.rpc.send_msg(msg)
 
+    def _notify_buy_fill(self, trade: Trade) -> None:
+        msg = {
+            'trade_id': trade.id,
+            'type': RPCMessageType.BUY_FILL,
+            'exchange': self.exchange.name.capitalize(),
+            'pair': trade.pair,
+            'open_rate': trade.open_rate,
+            'stake_amount': trade.stake_amount,
+            'stake_currency': self.config['stake_currency'],
+            'fiat_currency': self.config.get('fiat_display_currency', None),
+            'amount': trade.amount,
+            'open_date': trade.open_date,
+        }
+        self.rpc.send_msg(msg)
+
 #
 # SELL / exit positions / close trades logic and methods
 #
@@ -698,7 +714,7 @@ class FreqtradeBot(LoggingMixin):
             except DependencyException as exception:
                 logger.warning('Unable to sell trade %s: %s', trade.pair, exception)
 
-        # Updating wallets if any trade occured
+        # Updating wallets if any trade occurred
         if trades_closed:
             self.wallets.update()
 
@@ -835,7 +851,8 @@ class FreqtradeBot(LoggingMixin):
             trade.stoploss_order_id = None
             logger.error(f'Unable to place a stoploss order on exchange. {e}')
             logger.warning('Selling the trade forcefully')
-            self.execute_sell(trade, trade.stop_loss, sell_reason=SellType.EMERGENCY_SELL)
+            self.execute_sell(trade, trade.stop_loss, sell_reason=SellCheckTuple(
+                sell_type=SellType.EMERGENCY_SELL))
 
         except ExchangeError:
             trade.stoploss_order_id = None
@@ -916,14 +933,15 @@ class FreqtradeBot(LoggingMixin):
         :return: None
         """
         if self.exchange.stoploss_adjust(trade.stop_loss, order):
-            # we check if the update is neccesary
+            # we check if the update is necessary
             update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60)
             if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat:
                 # cancelling the current stoploss on exchange first
                 logger.info(f"Cancelling current stoploss on exchange for pair {trade.pair} "
                             f"(orderid:{order['id']}) in order to add another one ...")
                 try:
-                    co = self.exchange.cancel_stoploss_order(order['id'], trade.pair)
+                    co = self.exchange.cancel_stoploss_order_with_result(order['id'], trade.pair,
+                                                                         trade.amount)
                     trade.update_order(co)
                 except InvalidOrderException:
                     logger.exception(f"Could not cancel stoploss order {order['id']} "
@@ -946,7 +964,7 @@ class FreqtradeBot(LoggingMixin):
 
         if should_sell.sell_flag:
             logger.info(f'Executing Sell for {trade.pair}. Reason: {should_sell.sell_type}')
-            self.execute_sell(trade, sell_rate, should_sell.sell_type)
+            self.execute_sell(trade, sell_rate, should_sell)
             return True
         return False
 
@@ -957,15 +975,16 @@ class FreqtradeBot(LoggingMixin):
         timeout = self.config.get('unfilledtimeout', {}).get(side)
         ordertime = arrow.get(order['datetime']).datetime
         if timeout is not None:
-            timeout_threshold = arrow.utcnow().shift(minutes=-timeout).datetime
+            timeout_unit = self.config.get('unfilledtimeout', {}).get('unit', 'minutes')
-
+            timeout_kwargs = {timeout_unit: -timeout}
+            timeout_threshold = arrow.utcnow().shift(**timeout_kwargs).datetime
             return (order['status'] == 'open' and order['side'] == side
                     and ordertime < timeout_threshold)
         return False
 
     def check_handle_timedout(self) -> None:
         """
-        Check if any orders are timed out and cancel if neccessary
+        Check if any orders are timed out and cancel if necessary
         :param timeoutvalue: Number of minutes until order is considered timed out
         :return: None
         """
@@ -1027,6 +1046,16 @@ class FreqtradeBot(LoggingMixin):
 
         # Cancelled orders may have the status of 'canceled' or 'closed'
         if order['status'] not in ('cancelled', 'canceled', 'closed'):
+            filled_val = order.get('filled', 0.0) or 0.0
+            filled_stake = filled_val * trade.open_rate
+            minstake = self.exchange.get_min_pair_stake_amount(
+                trade.pair, trade.open_rate, self.strategy.stoploss)
+
+            if filled_val > 0 and filled_stake < minstake:
+                logger.warning(
+                    f"Order {trade.open_order_id} for {trade.pair} not cancelled, "
+                    f"as the filled amount of {filled_val} would result in an unsellable trade.")
+                return False
             corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
                                                             trade.amount)
             # Avoid race condition where the order could not be cancelled coz its already filled.
@@ -1135,16 +1164,16 @@ class FreqtradeBot(LoggingMixin):
             raise DependencyException(
                 f"Not enough amount to sell. Trade-amount: {amount}, Wallet: {wallet_amount}")
 
-    def execute_sell(self, trade: Trade, limit: float, sell_reason: SellType) -> bool:
+    def execute_sell(self, trade: Trade, limit: float, sell_reason: SellCheckTuple) -> bool:
         """
         Executes a limit sell for the given trade and limit
         :param trade: Trade instance
         :param limit: limit rate for the sell order
-        :param sellreason: Reason the sell was triggered
+        :param sell_reason: Reason the sell was triggered
         :return: True if it succeeds (supported) False (not supported)
         """
         sell_type = 'sell'
-        if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS):
+        if sell_reason.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS):
             sell_type = 'stoploss'
 
         # if stoploss is on exchange and we are on dry_run mode,
@@ -1156,15 +1185,17 @@ class FreqtradeBot(LoggingMixin):
         # First cancelling stoploss on exchange ...
         if self.strategy.order_types.get('stoploss_on_exchange') and trade.stoploss_order_id:
             try:
-                self.exchange.cancel_stoploss_order(trade.stoploss_order_id, trade.pair)
+                co = self.exchange.cancel_stoploss_order_with_result(trade.stoploss_order_id,
+                                                                     trade.pair, trade.amount)
+                trade.update_order(co)
             except InvalidOrderException:
                 logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id}")
 
         order_type = self.strategy.order_types[sell_type]
-        if sell_reason == SellType.EMERGENCY_SELL:
+        if sell_reason.sell_type == SellType.EMERGENCY_SELL:
             # Emergency sells (default to market!)
             order_type = self.strategy.order_types.get("emergencysell", "market")
-        if sell_reason == SellType.FORCE_SELL:
+        if sell_reason.sell_type == SellType.FORCE_SELL:
             # Force sells (default to the sell_type defined in the strategy,
             # but we allow this value to be changed)
             order_type = self.strategy.order_types.get("forcesell", order_type)
@@ -1174,8 +1205,8 @@ class FreqtradeBot(LoggingMixin):
 
         if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)(
                 pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit,
-                time_in_force=time_in_force,
+                time_in_force=time_in_force, sell_reason=sell_reason.sell_reason,
-                sell_reason=sell_reason.value):
+                current_time=datetime.now(timezone.utc)):
             logger.info(f"User requested abortion of selling {trade.pair}")
             return False
 
@@ -1198,13 +1229,13 @@ class FreqtradeBot(LoggingMixin):
         trade.open_order_id = order['id']
         trade.sell_order_status = ''
         trade.close_rate_requested = limit
-        trade.sell_reason = sell_reason.value
+        trade.sell_reason = sell_reason.sell_reason
         # In case of market sell orders the order can be closed immediately
         if order.get('status', 'unknown') == 'closed':
             self.update_trade_state(trade, trade.open_order_id, order)
         Trade.query.session.flush()
 
-        # Lock pair for one candle to prevent immediate rebuys
+        # Lock pair for one candle to prevent immediate re-buys
         self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
                                 reason='Auto lock')
 
@@ -1212,19 +1243,20 @@ class FreqtradeBot(LoggingMixin):
 
         return True
 
-    def _notify_sell(self, trade: Trade, order_type: str) -> None:
+    def _notify_sell(self, trade: Trade, order_type: str, fill: bool = False) -> None:
         """
-        Sends rpc notification when a sell occured.
+        Sends rpc notification when a sell occurred.
         """
         profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
         profit_trade = trade.calc_profit(rate=profit_rate)
         # Use cached rates here - it was updated seconds ago.
-        current_rate = self.get_sell_rate(trade.pair, False)
+        current_rate = self.get_sell_rate(trade.pair, False) if not fill else None
         profit_ratio = trade.calc_profit_ratio(profit_rate)
         gain = "profit" if profit_ratio > 0 else "loss"
 
         msg = {
-            'type': RPCMessageType.SELL_NOTIFICATION,
+            'type': (RPCMessageType.SELL_FILL if fill
+                     else RPCMessageType.SELL),
             'trade_id': trade.id,
             'exchange': trade.exchange.capitalize(),
             'pair': trade.pair,
@@ -1233,6 +1265,7 @@ class FreqtradeBot(LoggingMixin):
             'order_type': order_type,
             'amount': trade.amount,
             'open_rate': trade.open_rate,
+            'close_rate': trade.close_rate,
             'current_rate': current_rate,
             'profit_amount': profit_trade,
             'profit_ratio': profit_ratio,
@@ -1253,7 +1286,7 @@ class FreqtradeBot(LoggingMixin):
 
     def _notify_sell_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
         """
-        Sends rpc notification when a sell cancel occured.
+        Sends rpc notification when a sell cancel occurred.
         """
         if trade.sell_order_status == reason:
             return
@@ -1267,7 +1300,7 @@ class FreqtradeBot(LoggingMixin):
         gain = "profit" if profit_ratio > 0 else "loss"
 
         msg = {
-            'type': RPCMessageType.SELL_CANCEL_NOTIFICATION,
+            'type': RPCMessageType.SELL_CANCEL,
             'trade_id': trade.id,
             'exchange': trade.exchange.capitalize(),
             'pair': trade.pair,
@@ -1306,7 +1339,7 @@ class FreqtradeBot(LoggingMixin):
         Handles closing both buy and sell orders.
         :param trade: Trade object of the trade we're analyzing
         :param order_id: Order-id of the order we're analyzing
-        :param action_order: Already aquired order object
+        :param action_order: Already acquired order object
         :return: True if order has been cancelled without being filled partially, False otherwise
         """
         if not order_id:
@@ -1344,9 +1377,15 @@ class FreqtradeBot(LoggingMixin):
 
         # Updating wallets when order is closed
         if not trade.is_open:
+            if not stoploss_order and not trade.open_order_id:
+                self._notify_sell(trade, '', True)
             self.protections.stop_per_pair(trade.pair)
             self.protections.global_stop()
             self.wallets.update()
+        elif not trade.open_order_id:
+            # Buy fill
+            self._notify_buy_fill(trade)
+
         return False
 
     def apply_fee_conditional(self, trade: Trade, trade_base_currency: str,
@@ -1370,7 +1409,7 @@ class FreqtradeBot(LoggingMixin):
     def get_real_amount(self, trade: Trade, order: Dict) -> float:
         """
         Detect and update trade fee.
-        Calls trade.update_fee() uppon correct detection.
+        Calls trade.update_fee() upon correct detection.
         Returns modified amount if the fee was taken from the destination currency.
         Necessary for exchanges which charge fees in base currency (e.g. binance)
         :return: identical (or new) amount for the trade
@@ -1403,8 +1442,8 @@ class FreqtradeBot(LoggingMixin):
         """
         fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee.
         """
-        trades = self.exchange.get_trades_for_order(order['id'], trade.pair,
+        trades = self.exchange.get_trades_for_order(self.exchange.get_order_id_conditional(order),
-                                                    trade.open_date)
+                                                    trade.pair, trade.open_date)
 
         if len(trades) == 0:
             logger.info("Applying fee on amount for %s failed: myTrade-Dict empty found", trade)

freqtrade/misc.py

@@ -6,7 +6,7 @@ import logging
 import re
 from datetime import datetime
 from pathlib import Path
-from typing import Any
+from typing import Any, Iterator, List
 from typing.io import IO
 
 import rapidjson
@@ -202,3 +202,14 @@ def render_template_with_fallback(templatefile: str, templatefallbackfile: str,
         return render_template(templatefile, arguments)
     except TemplateNotFound:
         return render_template(templatefallbackfile, arguments)
+
+
+def chunks(lst: List[Any], n: int) -> Iterator[List[Any]]:
+    """
+    Split lst into chunks of the size n.
+    :param lst: list to split into chunks
+    :param n: number of max elements per chunk
+    :return: None
+    """
+    for chunk in range(0, len(lst), n):
+        yield (lst[chunk:chunk + n])

freqtrade/optimize/backtesting.py

@@ -15,7 +15,7 @@ from freqtrade.configuration import TimeRange, remove_credentials, validate_conf
 from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data import history
 from freqtrade.data.btanalysis import trade_list_to_dataframe
-from freqtrade.data.converter import trim_dataframe
+from freqtrade.data.converter import trim_dataframes
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
@@ -63,9 +63,7 @@ class Backtesting:
         self.all_results: Dict[str, Dict] = {}
 
         self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
-
+        self.dataprovider = DataProvider(self.config, None)
-        dataprovider = DataProvider(self.config, self.exchange)
-        IStrategy.dp = dataprovider
 
         if self.config.get('strategy_list', None):
             for strat in list(self.config['strategy_list']):
@@ -96,7 +94,7 @@ class Backtesting:
                 "PrecisionFilter not allowed for backtesting multiple strategies."
             )
 
-        dataprovider.add_pairlisthandler(self.pairlists)
+        self.dataprovider.add_pairlisthandler(self.pairlists)
         self.pairlists.refresh_pairlist()
 
         if len(self.pairlists.whitelist) == 0:
@@ -112,15 +110,11 @@ class Backtesting:
         PairLocks.timeframe = self.config['timeframe']
         PairLocks.use_db = False
         PairLocks.reset_locks()
-        if self.config.get('enable_protections', False):
-            self.protections = ProtectionManager(self.config)
 
         self.wallets = Wallets(self.config, self.exchange, log=False)
 
         # Get maximum required startup period
         self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
-        # Load one (first) strategy
-        self._set_strategy(self.strategylist[0])
 
     def __del__(self):
         LoggingMixin.show_output = True
@@ -132,10 +126,17 @@ class Backtesting:
         Load strategy into backtesting
         """
         self.strategy: IStrategy = strategy
+        strategy.dp = self.dataprovider
         # Set stoploss_on_exchange to false for backtesting,
         # since a "perfect" stoploss-sell is assumed anyway
         # And the regular "stoploss" function would not apply to that case
         self.strategy.order_types['stoploss_on_exchange'] = False
+        if self.config.get('enable_protections', False):
+            conf = self.config
+            if hasattr(strategy, 'protections'):
+                conf = deepcopy(conf)
+                conf['protections'] = strategy.protections
+            self.protections = ProtectionManager(conf)
 
     def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]:
         """
@@ -159,7 +160,7 @@ class Backtesting:
 
         logger.info(f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
-                    f'({(max_date - min_date).days} days)..')
+                    f'({(max_date - min_date).days} days).')
 
         # Adjust startts forward if not enough data is available
         timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
@@ -176,6 +177,8 @@ class Backtesting:
         Trade.use_db = False
         PairLocks.reset_locks()
         Trade.reset_trades()
+        self.rejected_trades = 0
+        self.dataprovider.clear_cache()
 
     def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]:
         """
@@ -189,8 +192,9 @@ class Backtesting:
         data: Dict = {}
         # Create dict with data
         for pair, pair_data in processed.items():
-            pair_data.loc[:, 'buy'] = 0  # cleanup from previous run
+            if not pair_data.empty:
-            pair_data.loc[:, 'sell'] = 0  # cleanup from previous run
+                pair_data.loc[:, 'buy'] = 0  # cleanup if buy_signal is exist
+                pair_data.loc[:, 'sell'] = 0  # cleanup if sell_signal is exist
 
             df_analyzed = self.strategy.advise_sell(
                 self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
@@ -214,6 +218,12 @@ class Backtesting:
         """
         # Special handling if high or low hit STOP_LOSS or ROI
         if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS):
+            if trade.stop_loss > sell_row[HIGH_IDX]:
+                # our stoploss was already higher than candle high,
+                # possibly due to a cancelled trade exit.
+                # sell at open price.
+                return sell_row[OPEN_IDX]
+
             # Set close_rate to stoploss
             return trade.stop_loss
         elif sell.sell_type == (SellType.ROI):
@@ -239,7 +249,7 @@ class Backtesting:
                 # Use the maximum between close_rate and low as we
                 # cannot sell outside of a candle.
                 # Applies when a new ROI setting comes in place and the whole candle is above that.
-                return max(close_rate, sell_row[LOW_IDX])
+                return min(max(close_rate, sell_row[LOW_IDX]), sell_row[HIGH_IDX])
 
             else:
                 # This should not be reached...
@@ -250,12 +260,13 @@ class Backtesting:
     def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]:
 
         sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX],  # type: ignore
-                                         sell_row[DATE_IDX], sell_row[BUY_IDX], sell_row[SELL_IDX],
+                                         sell_row[DATE_IDX].to_pydatetime(), sell_row[BUY_IDX],
+                                         sell_row[SELL_IDX],
                                          low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX])
 
         if sell.sell_flag:
-            trade.close_date = sell_row[DATE_IDX]
+            trade.close_date = sell_row[DATE_IDX].to_pydatetime()
-            trade.sell_reason = sell.sell_type.value
+            trade.sell_reason = sell.sell_reason
             trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60)
             closerate = self._get_close_rate(sell_row, trade, sell, trade_dur)
 
@@ -265,7 +276,8 @@ class Backtesting:
                     pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount,
                     rate=closerate,
                     time_in_force=time_in_force,
-                    sell_reason=sell.sell_type.value):
+                    sell_reason=sell.sell_reason,
+                    current_time=sell_row[DATE_IDX].to_pydatetime()):
                 return None
 
             trade.close(closerate, show_msg=False)
@@ -273,11 +285,9 @@ class Backtesting:
 
         return None
 
-    def _enter_trade(self, pair: str, row: List, max_open_trades: int,
+    def _enter_trade(self, pair: str, row: List) -> Optional[LocalTrade]:
-                     open_trade_count: int) -> Optional[LocalTrade]:
         try:
-            stake_amount = self.wallets.get_trade_stake_amount(
+            stake_amount = self.wallets.get_trade_stake_amount(pair, None)
-                pair, max_open_trades - open_trade_count, None)
         except DependencyException:
             return None
         min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05)
@@ -287,7 +297,7 @@ class Backtesting:
         # Confirm trade entry:
         if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)(
                 pair=pair, order_type=order_type, amount=stake_amount, rate=row[OPEN_IDX],
-                time_in_force=time_in_force):
+                time_in_force=time_in_force, current_time=row[DATE_IDX].to_pydatetime()):
             return None
 
         if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount):
@@ -295,7 +305,7 @@ class Backtesting:
             trade = LocalTrade(
                 pair=pair,
                 open_rate=row[OPEN_IDX],
-                open_date=row[DATE_IDX],
+                open_date=row[DATE_IDX].to_pydatetime(),
                 stake_amount=stake_amount,
                 amount=round(stake_amount / row[OPEN_IDX], 8),
                 fee_open=self.fee,
@@ -317,7 +327,7 @@ class Backtesting:
                 for trade in open_trades[pair]:
                     sell_row = data[pair][-1]
 
-                    trade.close_date = sell_row[DATE_IDX]
+                    trade.close_date = sell_row[DATE_IDX].to_pydatetime()
                     trade.sell_reason = SellType.FORCE_SELL.value
                     trade.close(sell_row[OPEN_IDX], show_msg=False)
                     LocalTrade.close_bt_trade(trade)
@@ -327,10 +337,18 @@ class Backtesting:
                     trades.append(trade1)
         return trades
 
+    def trade_slot_available(self, max_open_trades: int, open_trade_count: int) -> bool:
+        # Always allow trades when max_open_trades is enabled.
+        if max_open_trades <= 0 or open_trade_count < max_open_trades:
+            return True
+        # Rejected trade
+        self.rejected_trades += 1
+        return False
+
     def backtest(self, processed: Dict,
                  start_date: datetime, end_date: datetime,
                  max_open_trades: int = 0, position_stacking: bool = False,
-                 enable_protections: bool = False) -> DataFrame:
+                 enable_protections: bool = False) -> Dict[str, Any]:
         """
         Implement backtesting functionality
 
@@ -349,12 +367,16 @@ class Backtesting:
         trades: List[LocalTrade] = []
         self.prepare_backtest(enable_protections)
 
+        # Update dataprovider cache
+        for pair, dataframe in processed.items():
+            self.dataprovider._set_cached_df(pair, self.timeframe, dataframe)
+
         # Use dict of lists with data for performance
         # (looping lists is a lot faster than pandas DataFrames)
         data: Dict = self._get_ohlcv_as_lists(processed)
 
         # Indexes per pair, so some pairs are allowed to have a missing start.
-        indexes: Dict = {}
+        indexes: Dict = defaultdict(int)
         tmp = start_date + timedelta(minutes=self.timeframe_min)
 
         open_trades: Dict[str, List[LocalTrade]] = defaultdict(list)
@@ -365,11 +387,9 @@ class Backtesting:
             open_trade_count_start = open_trade_count
 
             for i, pair in enumerate(data):
-                if pair not in indexes:
+                row_index = indexes[pair]
-                    indexes[pair] = 0
-
                 try:
-                    row = data[pair][indexes[pair]]
+                    row = data[pair][row_index]
                 except IndexError:
                     # missing Data for one pair at the end.
                     # Warnings for this are shown during data loading
@@ -378,17 +398,23 @@ class Backtesting:
                 # Waits until the time-counter reaches the start of the data for this pair.
                 if row[DATE_IDX] > tmp:
                     continue
-                indexes[pair] += 1
+
+                row_index += 1
+                self.dataprovider._set_dataframe_max_index(row_index)
+                indexes[pair] = row_index
 
                 # without positionstacking, we can only have one open trade per pair.
                 # max_open_trades must be respected
                 # don't open on the last row
-                if ((position_stacking or len(open_trades[pair]) == 0)
+                if (
-                        and (max_open_trades <= 0 or open_trade_count_start < max_open_trades)
+                    (position_stacking or len(open_trades[pair]) == 0)
+                    and self.trade_slot_available(max_open_trades, open_trade_count_start)
                     and tmp != end_date
-                        and row[BUY_IDX] == 1 and row[SELL_IDX] != 1
+                    and row[BUY_IDX] == 1
-                        and not PairLocks.is_pair_locked(pair, row[DATE_IDX])):
+                    and row[SELL_IDX] != 1
-                    trade = self._enter_trade(pair, row, max_open_trades, open_trade_count_start)
+                    and not PairLocks.is_pair_locked(pair, row[DATE_IDX])
+                ):
+                    trade = self._enter_trade(pair, row)
                     if trade:
                         # TODO: hacky workaround to avoid opening > max_open_trades
                         # This emulates previous behaviour - not sure if this is correct
@@ -420,7 +446,14 @@ class Backtesting:
         trades += self.handle_left_open(open_trades, data=data)
         self.wallets.update()
 
-        return trade_list_to_dataframe(trades)
+        results = trade_list_to_dataframe(trades)
+        return {
+            'results': results,
+            'config': self.strategy.config,
+            'locks': PairLocks.get_all_locks(),
+            'rejected_signals': self.rejected_trades,
+            'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']),
+        }
 
     def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange):
         logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
@@ -442,32 +475,32 @@ class Backtesting:
         preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
 
         # Trim startup period from analyzed dataframe
-        for pair, df in preprocessed.items():
+        preprocessed = trim_dataframes(preprocessed, timerange, self.required_startup)
-            preprocessed[pair] = trim_dataframe(df, timerange,
-                                                startup_candles=self.required_startup)
-        min_date, max_date = history.get_timerange(preprocessed)
 
+        if not preprocessed:
+            raise OperationalException(
+                "No data left after adjusting for startup candles.")
+
+        min_date, max_date = history.get_timerange(preprocessed)
         logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
-                    f'({(max_date - min_date).days} days)..')
+                    f'({(max_date - min_date).days} days).')
         # Execute backtest and store results
         results = self.backtest(
             processed=preprocessed,
-            start_date=min_date.datetime,
+            start_date=min_date,
-            end_date=max_date.datetime,
+            end_date=max_date,
             max_open_trades=max_open_trades,
             position_stacking=self.config.get('position_stacking', False),
             enable_protections=self.config.get('enable_protections', False),
         )
         backtest_end_time = datetime.now(timezone.utc)
-        self.all_results[self.strategy.get_strategy_name()] = {
+        results.update({
-            'results': results,
-            'config': self.strategy.config,
-            'locks': PairLocks.get_all_locks(),
-            'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']),
             'backtest_start_time': int(backtest_start_time.timestamp()),
             'backtest_end_time': int(backtest_end_time.timestamp()),
-        }
+        })
+        self.all_results[self.strategy.get_strategy_name()] = results
+
         return min_date, max_date
 
     def start(self) -> None:
@@ -478,6 +511,7 @@ class Backtesting:
         data: Dict[str, Any] = {}
 
         data, timerange = self.load_bt_data()
+        logger.info("Dataload complete. Calculating indicators")
 
         for strat in self.strategylist:
             min_date, max_date = self.backtest_one_strategy(strat, data, timerange)

freqtrade/optimize/hyperopt.py

@@ -4,24 +4,24 @@
 This module contains the hyperopt logic
 """
 
-import locale
 import logging
 import random
 import warnings
-from datetime import datetime
+from datetime import datetime, timezone
 from math import ceil
-from operator import itemgetter
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
+import numpy as np
 import progressbar
+import rapidjson
 from colorama import Fore, Style
 from colorama import init as colorama_init
 from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects
 from pandas import DataFrame
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN
-from freqtrade.data.converter import trim_dataframe
+from freqtrade.data.converter import trim_dataframes
 from freqtrade.data.history import get_timerange
 from freqtrade.misc import file_dump_json, plural
 from freqtrade.optimize.backtesting import Backtesting
@@ -30,8 +30,8 @@ from freqtrade.optimize.hyperopt_auto import HyperOptAuto
 from freqtrade.optimize.hyperopt_interface import IHyperOpt  # noqa: F401
 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss  # noqa: F401
 from freqtrade.optimize.hyperopt_tools import HyperoptTools
+from freqtrade.optimize.optimize_reports import generate_strategy_stats
 from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver
-from freqtrade.strategy import IStrategy
 
 
 # Suppress scikit-learn FutureWarnings from skopt
@@ -65,6 +65,13 @@ class Hyperopt:
     custom_hyperopt: IHyperOpt
 
     def __init__(self, config: Dict[str, Any]) -> None:
+        self.buy_space: List[Dimension] = []
+        self.sell_space: List[Dimension] = []
+        self.roi_space: List[Dimension] = []
+        self.stoploss_space: List[Dimension] = []
+        self.trailing_space: List[Dimension] = []
+        self.dimensions: List[Dimension] = []
+
         self.config = config
 
         self.backtesting = Backtesting(self.config)
@@ -73,15 +80,15 @@ class Hyperopt:
             self.custom_hyperopt = HyperOptAuto(self.config)
         else:
             self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config)
+        self.backtesting._set_strategy(self.backtesting.strategylist[0])
         self.custom_hyperopt.strategy = self.backtesting.strategy
 
         self.custom_hyperoptloss = HyperOptLossResolver.load_hyperoptloss(self.config)
         self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function
         time_now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
         strategy = str(self.config['strategy'])
-        self.results_file = (self.config['user_data_dir'] /
+        self.results_file: Path = (self.config['user_data_dir'] / 'hyperopt_results' /
-                             'hyperopt_results' /
+                                   f'strategy_{strategy}_{time_now}.fthypt')
-                             f'strategy_{strategy}_hyperopt_results_{time_now}.pickle')
         self.data_pickle_file = (self.config['user_data_dir'] /
                                  'hyperopt_results' / 'hyperopt_tickerdata.pkl')
         self.total_epochs = config.get('epochs', 0)
@@ -91,9 +98,7 @@ class Hyperopt:
         self.clean_hyperopt()
 
         self.num_epochs_saved = 0
-
+        self.current_best_epoch: Optional[Dict[str, Any]] = None
-        # Previous evaluations
-        self.epochs: List = []
 
         # Populate functions here (hasattr is slow so should not be run during "regular" operations)
         if hasattr(self.custom_hyperopt, 'populate_indicators'):
@@ -114,7 +119,7 @@ class Hyperopt:
             self.max_open_trades = 0
         self.position_stacking = self.config.get('position_stacking', False)
 
-        if self.has_space('sell'):
+        if HyperoptTools.has_space(self.config, 'sell'):
             # Make sure use_sell_signal is enabled
             if 'ask_strategy' not in self.config:
                 self.config['ask_strategy'] = {}
@@ -140,9 +145,7 @@ class Hyperopt:
                 logger.info(f"Removing `{p}`.")
                 p.unlink()
 
-    def _get_params_dict(self, raw_params: List[Any]) -> Dict:
+    def _get_params_dict(self, dimensions: List[Dimension], raw_params: List[Any]) -> Dict:
-
-        dimensions: List[Dimension] = self.dimensions
 
         # Ensure the number of dimensions match
         # the number of parameters in the list.
@@ -153,15 +156,24 @@ class Hyperopt:
         # and the values are taken from the list of parameters.
         return {d.name: v for d, v in zip(dimensions, raw_params)}
 
-    def _save_results(self) -> None:
+    def _save_result(self, epoch: Dict) -> None:
         """
         Save hyperopt results to file
+        Store one line per epoch.
+        While not a valid json object - this allows appending easily.
+        :param epoch: result dictionary for this epoch.
         """
-        num_epochs = len(self.epochs)
+        def default_parser(x):
-        if num_epochs > self.num_epochs_saved:
+            if isinstance(x, np.integer):
-            logger.debug(f"Saving {num_epochs} {plural(num_epochs, 'epoch')}.")
+                return int(x)
-            dump(self.epochs, self.results_file)
+            return str(x)
-            self.num_epochs_saved = num_epochs
+
+        with self.results_file.open('a') as f:
+            rapidjson.dump(epoch, f, default=default_parser,
+                           number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN)
+            f.write("\n")
+
+        self.num_epochs_saved += 1
         logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
                      f"saved to '{self.results_file}'.")
         # Store hyperopt filename
@@ -175,18 +187,16 @@ class Hyperopt:
         """
         result: Dict = {}
 
-        if self.has_space('buy'):
+        if HyperoptTools.has_space(self.config, 'buy'):
-            result['buy'] = {p.name: params.get(p.name)
+            result['buy'] = {p.name: params.get(p.name) for p in self.buy_space}
-                             for p in self.hyperopt_space('buy')}
+        if HyperoptTools.has_space(self.config, 'sell'):
-        if self.has_space('sell'):
+            result['sell'] = {p.name: params.get(p.name) for p in self.sell_space}
-            result['sell'] = {p.name: params.get(p.name)
+        if HyperoptTools.has_space(self.config, 'roi'):
-                              for p in self.hyperopt_space('sell')}
+            result['roi'] = {str(k): v for k, v in
-        if self.has_space('roi'):
+                             self.custom_hyperopt.generate_roi_table(params).items()}
-            result['roi'] = self.custom_hyperopt.generate_roi_table(params)
+        if HyperoptTools.has_space(self.config, 'stoploss'):
-        if self.has_space('stoploss'):
+            result['stoploss'] = {p.name: params.get(p.name) for p in self.stoploss_space}
-            result['stoploss'] = {p.name: params.get(p.name)
+        if HyperoptTools.has_space(self.config, 'trailing'):
-                                  for p in self.hyperopt_space('stoploss')}
-        if self.has_space('trailing'):
             result['trailing'] = self.custom_hyperopt.generate_trailing_params(params)
 
         return result
@@ -208,71 +218,58 @@ class Hyperopt:
             )
             self.hyperopt_table_header = 2
 
-    def has_space(self, space: str) -> bool:
+    def init_spaces(self):
         """
-        Tell if the space value is contained in the configuration
+        Assign the dimensions in the hyperoptimization space.
         """
-        # The 'trailing' space is not included in the 'default' set of spaces
-        if space == 'trailing':
-            return any(s in self.config['spaces'] for s in [space, 'all'])
-        else:
-            return any(s in self.config['spaces'] for s in [space, 'all', 'default'])
 
-    def hyperopt_space(self, space: Optional[str] = None) -> List[Dimension]:
+        if HyperoptTools.has_space(self.config, 'buy'):
-        """
-        Return the dimensions in the hyperoptimization space.
-        :param space: Defines hyperspace to return dimensions for.
-        If None, then the self.has_space() will be used to return dimensions
-        for all hyperspaces used.
-        """
-        spaces: List[Dimension] = []
-
-        if space == 'buy' or (space is None and self.has_space('buy')):
             logger.debug("Hyperopt has 'buy' space")
-            spaces += self.custom_hyperopt.indicator_space()
+            self.buy_space = self.custom_hyperopt.indicator_space()
 
-        if space == 'sell' or (space is None and self.has_space('sell')):
+        if HyperoptTools.has_space(self.config, 'sell'):
             logger.debug("Hyperopt has 'sell' space")
-            spaces += self.custom_hyperopt.sell_indicator_space()
+            self.sell_space = self.custom_hyperopt.sell_indicator_space()
 
-        if space == 'roi' or (space is None and self.has_space('roi')):
+        if HyperoptTools.has_space(self.config, 'roi'):
             logger.debug("Hyperopt has 'roi' space")
-            spaces += self.custom_hyperopt.roi_space()
+            self.roi_space = self.custom_hyperopt.roi_space()
 
-        if space == 'stoploss' or (space is None and self.has_space('stoploss')):
+        if HyperoptTools.has_space(self.config, 'stoploss'):
             logger.debug("Hyperopt has 'stoploss' space")
-            spaces += self.custom_hyperopt.stoploss_space()
+            self.stoploss_space = self.custom_hyperopt.stoploss_space()
 
-        if space == 'trailing' or (space is None and self.has_space('trailing')):
+        if HyperoptTools.has_space(self.config, 'trailing'):
             logger.debug("Hyperopt has 'trailing' space")
-            spaces += self.custom_hyperopt.trailing_space()
+            self.trailing_space = self.custom_hyperopt.trailing_space()
-
+        self.dimensions = (self.buy_space + self.sell_space + self.roi_space +
-        return spaces
+                           self.stoploss_space + self.trailing_space)
 
     def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict:
         """
         Used Optimize function. Called once per epoch to optimize whatever is configured.
         Keep this function as optimized as possible!
         """
-        params_dict = self._get_params_dict(raw_params)
+        backtest_start_time = datetime.now(timezone.utc)
-        params_details = self._get_params_details(params_dict)
+        params_dict = self._get_params_dict(self.dimensions, raw_params)
 
-        if self.has_space('roi'):
+        # Apply parameters
+        if HyperoptTools.has_space(self.config, 'roi'):
             self.backtesting.strategy.minimal_roi = (  # type: ignore
                 self.custom_hyperopt.generate_roi_table(params_dict))
 
-        if self.has_space('buy'):
+        if HyperoptTools.has_space(self.config, 'buy'):
             self.backtesting.strategy.advise_buy = (  # type: ignore
                 self.custom_hyperopt.buy_strategy_generator(params_dict))
 
-        if self.has_space('sell'):
+        if HyperoptTools.has_space(self.config, 'sell'):
             self.backtesting.strategy.advise_sell = (  # type: ignore
                 self.custom_hyperopt.sell_strategy_generator(params_dict))
 
-        if self.has_space('stoploss'):
+        if HyperoptTools.has_space(self.config, 'stoploss'):
             self.backtesting.strategy.stoploss = params_dict['stoploss']
 
-        if self.has_space('trailing'):
+        if HyperoptTools.has_space(self.config, 'trailing'):
             d = self.custom_hyperopt.generate_trailing_params(params_dict)
             self.backtesting.strategy.trailing_stop = d['trailing_stop']
             self.backtesting.strategy.trailing_stop_positive = d['trailing_stop_positive']
@@ -281,30 +278,42 @@ class Hyperopt:
             self.backtesting.strategy.trailing_only_offset_is_reached = \
                 d['trailing_only_offset_is_reached']
 
-        processed = load(self.data_pickle_file)
+        with self.data_pickle_file.open('rb') as f:
-
+            processed = load(f, mmap_mode='r')
-        min_date, max_date = get_timerange(processed)
+        bt_results = self.backtesting.backtest(
-
-        backtesting_results = self.backtesting.backtest(
             processed=processed,
-            start_date=min_date.datetime,
+            start_date=self.min_date,
-            end_date=max_date.datetime,
+            end_date=self.max_date,
             max_open_trades=self.max_open_trades,
             position_stacking=self.position_stacking,
             enable_protections=self.config.get('enable_protections', False),
-
         )
-        return self._get_results_dict(backtesting_results, min_date, max_date,
+        backtest_end_time = datetime.now(timezone.utc)
-                                      params_dict, params_details,
+        bt_results.update({
+            'backtest_start_time': int(backtest_start_time.timestamp()),
+            'backtest_end_time': int(backtest_end_time.timestamp()),
+        })
+
+        return self._get_results_dict(bt_results, self.min_date, self.max_date,
+                                      params_dict,
                                       processed=processed)
 
     def _get_results_dict(self, backtesting_results, min_date, max_date,
-                          params_dict, params_details, processed: Dict[str, DataFrame]):
+                          params_dict, processed: Dict[str, DataFrame]
-        results_metrics = self._calculate_results_metrics(backtesting_results)
+                          ) -> Dict[str, Any]:
-        results_explanation = self._format_results_explanation_string(results_metrics)
+        params_details = self._get_params_details(params_dict)
 
-        trade_count = results_metrics['trade_count']
+        strat_stats = generate_strategy_stats(
-        total_profit = results_metrics['total_profit']
+            processed, self.backtesting.strategy.get_strategy_name(),
+            backtesting_results, min_date, max_date, market_change=0
+        )
+        results_explanation = HyperoptTools.format_results_explanation_string(
+            strat_stats, self.config['stake_currency'])
+
+        not_optimized = self.backtesting.strategy.get_params_dict()
+
+        trade_count = strat_stats['total_trades']
+        total_profit = strat_stats['profit_total']
 
         # If this evaluation contains too short amount of trades to be
         # interesting -- consider it as 'bad' (assigned max. loss value)
@@ -312,50 +321,20 @@ class Hyperopt:
         # path. We do not want to optimize 'hodl' strategies.
         loss: float = MAX_LOSS
         if trade_count >= self.config['hyperopt_min_trades']:
-            loss = self.calculate_loss(results=backtesting_results, trade_count=trade_count,
+            loss = self.calculate_loss(results=backtesting_results['results'],
-                                       min_date=min_date.datetime, max_date=max_date.datetime,
+                                       trade_count=trade_count,
+                                       min_date=min_date, max_date=max_date,
                                        config=self.config, processed=processed)
         return {
             'loss': loss,
             'params_dict': params_dict,
             'params_details': params_details,
-            'results_metrics': results_metrics,
+            'params_not_optimized': not_optimized,
+            'results_metrics': strat_stats,
             'results_explanation': results_explanation,
             'total_profit': total_profit,
         }
 
-    def _calculate_results_metrics(self, backtesting_results: DataFrame) -> Dict:
-        wins = len(backtesting_results[backtesting_results['profit_ratio'] > 0])
-        draws = len(backtesting_results[backtesting_results['profit_ratio'] == 0])
-        losses = len(backtesting_results[backtesting_results['profit_ratio'] < 0])
-        return {
-            'trade_count': len(backtesting_results.index),
-            'wins': wins,
-            'draws': draws,
-            'losses': losses,
-            'winsdrawslosses': f"{wins:>4} {draws:>4} {losses:>4}",
-            'avg_profit': backtesting_results['profit_ratio'].mean() * 100.0,
-            'median_profit': backtesting_results['profit_ratio'].median() * 100.0,
-            'total_profit': backtesting_results['profit_abs'].sum(),
-            'profit': backtesting_results['profit_ratio'].sum() * 100.0,
-            'duration': backtesting_results['trade_duration'].mean(),
-        }
-
-    def _format_results_explanation_string(self, results_metrics: Dict) -> str:
-        """
-        Return the formatted results explanation in a string
-        """
-        stake_cur = self.config['stake_currency']
-        return (f"{results_metrics['trade_count']:6d} trades. "
-                f"{results_metrics['wins']}/{results_metrics['draws']}"
-                f"/{results_metrics['losses']} Wins/Draws/Losses. "
-                f"Avg profit {results_metrics['avg_profit']: 6.2f}%. "
-                f"Median profit {results_metrics['median_profit']: 6.2f}%. "
-                f"Total profit {results_metrics['total_profit']: 11.8f} {stake_cur} "
-                f"({results_metrics['profit']: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). "
-                f"Avg duration {results_metrics['duration']:5.1f} min."
-                ).encode(locale.getpreferredencoding(), 'replace').decode('utf-8')
-
     def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
         return Optimizer(
             dimensions,
@@ -374,25 +353,31 @@ class Hyperopt:
     def _set_random_state(self, random_state: Optional[int]) -> int:
         return random_state or random.randint(1, 2**16 - 1)
 
-    def start(self) -> None:
+    def prepare_hyperopt_data(self) -> None:
-        self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
-        logger.info(f"Using optimizer random state: {self.random_state}")
-        self.hyperopt_table_header = -1
         data, timerange = self.backtesting.load_bt_data()
+        logger.info("Dataload complete. Calculating indicators")
 
         preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
 
         # Trim startup period from analyzed dataframe
-        for pair, df in preprocessed.items():
+        processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup)
-            preprocessed[pair] = trim_dataframe(df, timerange,
-                                                startup_candles=self.backtesting.required_startup)
-        min_date, max_date = get_timerange(preprocessed)
 
-        logger.info(f'Hyperopting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
+        self.min_date, self.max_date = get_timerange(processed)
-                    f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
-                    f'({(max_date - min_date).days} days)..')
 
-        dump(preprocessed, self.data_pickle_file)
+        logger.info(f'Hyperopting with data from {self.min_date.strftime(DATETIME_PRINT_FORMAT)} '
+                    f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} '
+                    f'({(self.max_date - self.min_date).days} days)..')
+
+        dump(processed, self.data_pickle_file)
+
+    def start(self) -> None:
+        self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
+        logger.info(f"Using optimizer random state: {self.random_state}")
+        self.hyperopt_table_header = -1
+        # Initialize spaces ...
+        self.init_spaces()
+
+        self.prepare_hyperopt_data()
 
         # We don't need exchange instance anymore while running hyperopt
         self.backtesting.exchange.close()
@@ -400,15 +385,12 @@ class Hyperopt:
         self.backtesting.exchange._api_async = None  # type: ignore
         # self.backtesting.exchange = None  # type: ignore
         self.backtesting.pairlists = None  # type: ignore
-        self.backtesting.strategy.dp = None  # type: ignore
-        IStrategy.dp = None  # type: ignore
 
         cpus = cpu_count()
         logger.info(f"Found {cpus} CPU cores. Let's make them scream!")
         config_jobs = self.config.get('hyperopt_jobs', -1)
         logger.info(f'Number of parallel jobs set as: {config_jobs}')
 
-        self.dimensions: List[Dimension] = self.hyperopt_space()
         self.opt = self.get_optimizer(self.dimensions, config_jobs)
 
         if self.print_colorized:
@@ -474,25 +456,21 @@ class Hyperopt:
 
                             if is_best:
                                 self.current_best_loss = val['loss']
-                            self.epochs.append(val)
+                                self.current_best_epoch = val
 
-                            # Save results after each best epoch and every 100 epochs
+                            self._save_result(val)
-                            if is_best or current % 100 == 0:
-                                self._save_results()
 
                             pbar.update(current)
 
         except KeyboardInterrupt:
             print('User interrupted..')
 
-        self._save_results()
         logger.info(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
                     f"saved to '{self.results_file}'.")
 
-        if self.epochs:
+        if self.current_best_epoch:
-            sorted_epochs = sorted(self.epochs, key=itemgetter('loss'))
+            HyperoptTools.print_epoch_details(self.current_best_epoch, self.total_epochs,
-            best_epoch = sorted_epochs[0]
+                                              self.print_json)
-            HyperoptTools.print_epoch_details(best_epoch, self.total_epochs, self.print_json)
         else:
             # This is printed when Ctrl+C is pressed quickly, before first epochs have
             # a chance to be evaluated.

freqtrade/optimize/hyperopt_auto.py

@@ -27,7 +27,7 @@ class HyperOptAuto(IHyperOpt):
             for attr_name, attr in self.strategy.enumerate_parameters('buy'):
                 if attr.optimize:
                     # noinspection PyProtectedMember
-                    attr._set_value(params[attr_name])
+                    attr.value = params[attr_name]
             return self.strategy.populate_buy_trend(dataframe, metadata)
 
         return populate_buy_trend
@@ -37,7 +37,7 @@ class HyperOptAuto(IHyperOpt):
             for attr_name, attr in self.strategy.enumerate_parameters('sell'):
                 if attr.optimize:
                     # noinspection PyProtectedMember
-                    attr._set_value(params[attr_name])
+                    attr.value = params[attr_name]
             return self.strategy.populate_sell_trend(dataframe, metadata)
 
         return populate_sell_trend

freqtrade/optimize/hyperopt_interface.py

@@ -7,11 +7,12 @@ import math
 from abc import ABC
 from typing import Any, Callable, Dict, List
 
-from skopt.space import Categorical, Dimension, Integer, Real
+from skopt.space import Categorical, Dimension, Integer
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import round_dict
+from freqtrade.optimize.space import SKDecimal
 from freqtrade.strategy import IStrategy
 
 
@@ -139,7 +140,7 @@ class IHyperOpt(ABC):
             'roi_p2': roi_limits['roi_p2_min'],
             'roi_p3': roi_limits['roi_p3_min'],
         }
-        logger.info(f"Min roi table: {round_dict(self.generate_roi_table(p), 5)}")
+        logger.info(f"Min roi table: {round_dict(self.generate_roi_table(p), 3)}")
         p = {
             'roi_t1': roi_limits['roi_t1_max'],
             'roi_t2': roi_limits['roi_t2_max'],
@@ -148,15 +149,18 @@ class IHyperOpt(ABC):
             'roi_p2': roi_limits['roi_p2_max'],
             'roi_p3': roi_limits['roi_p3_max'],
         }
-        logger.info(f"Max roi table: {round_dict(self.generate_roi_table(p), 5)}")
+        logger.info(f"Max roi table: {round_dict(self.generate_roi_table(p), 3)}")
 
         return [
             Integer(roi_limits['roi_t1_min'], roi_limits['roi_t1_max'], name='roi_t1'),
             Integer(roi_limits['roi_t2_min'], roi_limits['roi_t2_max'], name='roi_t2'),
             Integer(roi_limits['roi_t3_min'], roi_limits['roi_t3_max'], name='roi_t3'),
-            Real(roi_limits['roi_p1_min'], roi_limits['roi_p1_max'], name='roi_p1'),
+            SKDecimal(roi_limits['roi_p1_min'], roi_limits['roi_p1_max'], decimals=3,
-            Real(roi_limits['roi_p2_min'], roi_limits['roi_p2_max'], name='roi_p2'),
+                      name='roi_p1'),
-            Real(roi_limits['roi_p3_min'], roi_limits['roi_p3_max'], name='roi_p3'),
+            SKDecimal(roi_limits['roi_p2_min'], roi_limits['roi_p2_max'], decimals=3,
+                      name='roi_p2'),
+            SKDecimal(roi_limits['roi_p3_min'], roi_limits['roi_p3_max'], decimals=3,
+                      name='roi_p3'),
         ]
 
     def stoploss_space(self) -> List[Dimension]:
@@ -167,7 +171,7 @@ class IHyperOpt(ABC):
         You may override it in your custom Hyperopt class.
         """
         return [
-            Real(-0.35, -0.02, name='stoploss'),
+            SKDecimal(-0.35, -0.02, decimals=3, name='stoploss'),
         ]
 
     def generate_trailing_params(self, params: Dict) -> Dict:
@@ -197,14 +201,14 @@ class IHyperOpt(ABC):
             # other 'trailing' hyperspace parameters.
             Categorical([True], name='trailing_stop'),
 
-            Real(0.01, 0.35, name='trailing_stop_positive'),
+            SKDecimal(0.01, 0.35, decimals=3, name='trailing_stop_positive'),
 
             # 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive',
             # so this intermediate parameter is used as the value of the difference between
             # them. The value of the 'trailing_stop_positive_offset' is constructed in the
             # generate_trailing_params() method.
             # This is similar to the hyperspace dimensions used for constructing the ROI tables.
-            Real(0.001, 0.1, name='trailing_stop_positive_offset_p1'),
+            SKDecimal(0.001, 0.1, decimals=3, name='trailing_stop_positive_offset_p1'),
 
             Categorical([True, False], name='trailing_only_offset_is_reached'),
         ]

freqtrade/optimize/hyperopt_tools.py

@@ -1,19 +1,18 @@
 
 import io
+import locale
 import logging
 from collections import OrderedDict
 from pathlib import Path
-from pprint import pformat
+from typing import Any, Dict, List
-from typing import Dict, List
 
 import rapidjson
 import tabulate
 from colorama import Fore, Style
-from joblib import load
 from pandas import isna, json_normalize
 
 from freqtrade.exceptions import OperationalException
-from freqtrade.misc import round_dict
+from freqtrade.misc import round_coin_value, round_dict
 
 
 logger = logging.getLogger(__name__)
@@ -21,13 +20,38 @@ logger = logging.getLogger(__name__)
 
 class HyperoptTools():
 
+    @staticmethod
+    def has_space(config: Dict[str, Any], space: str) -> bool:
+        """
+        Tell if the space value is contained in the configuration
+        """
+        # The 'trailing' space is not included in the 'default' set of spaces
+        if space == 'trailing':
+            return any(s in config['spaces'] for s in [space, 'all'])
+        else:
+            return any(s in config['spaces'] for s in [space, 'all', 'default'])
+
+    @staticmethod
+    def _read_results_pickle(results_file: Path) -> List:
+        """
+        Read hyperopt results from pickle file
+        LEGACY method - new files are written as json and cannot be read with this method.
+        """
+        from joblib import load
+
+        logger.info(f"Reading pickled epochs from '{results_file}'")
+        data = load(results_file)
+        return data
+
     @staticmethod
     def _read_results(results_file: Path) -> List:
         """
         Read hyperopt results from file
         """
-        logger.info("Reading epochs from '%s'", results_file)
+        import rapidjson
-        data = load(results_file)
+        logger.info(f"Reading epochs from '{results_file}'")
+        with results_file.open('r') as f:
+            data = [rapidjson.loads(line) for line in f]
         return data
 
     @staticmethod
@@ -37,6 +61,9 @@ class HyperoptTools():
         """
         epochs: List = []
         if results_file.is_file() and results_file.stat().st_size > 0:
+            if results_file.suffix == '.pickle':
+                epochs = HyperoptTools._read_results_pickle(results_file)
+            else:
                 epochs = HyperoptTools._read_results(results_file)
             # Detection of some old format, without 'is_best' field saved
             if epochs[0].get('is_best') is None:
@@ -53,6 +80,7 @@ class HyperoptTools():
         Display details of the hyperopt result
         """
         params = results.get('params_details', {})
+        non_optimized = results.get('params_not_optimized', {})
 
         # Default header string
         if header_str is None:
@@ -69,8 +97,10 @@ class HyperoptTools():
             print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE))
 
         else:
-            HyperoptTools._params_pretty_print(params, 'buy', "Buy hyperspace params:")
+            HyperoptTools._params_pretty_print(params, 'buy', "Buy hyperspace params:",
-            HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:")
+                                               non_optimized)
+            HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:",
+                                               non_optimized)
             HyperoptTools._params_pretty_print(params, 'roi', "ROI table:")
             HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:")
             HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:")
@@ -96,12 +126,12 @@ class HyperoptTools():
                 result_dict.update(space_params)
 
     @staticmethod
-    def _params_pretty_print(params, space: str, header: str) -> None:
+    def _params_pretty_print(params, space: str, header: str, non_optimized={}) -> None:
-        if space in params:
+        if space in params or space in non_optimized:
             space_params = HyperoptTools._space_params(params, space, 5)
-            params_result = f"\n# {header}\n"
+            result = f"\n# {header}\n"
             if space == 'stoploss':
-                params_result += f"stoploss = {space_params.get('stoploss')}"
+                result += f"stoploss = {space_params.get('stoploss')}"
             elif space == 'roi':
                 # TODO: get rid of OrderedDict when support for python 3.6 will be
                 # dropped (dicts keep the order as the language feature)
@@ -110,28 +140,64 @@ class HyperoptTools():
                         (str(k), v) for k, v in space_params.items()
                     ),
                     default=str, indent=4, number_mode=rapidjson.NM_NATIVE)
-                params_result += f"minimal_roi = {minimal_roi_result}"
+                result += f"minimal_roi = {minimal_roi_result}"
             elif space == 'trailing':
 
                 for k, v in space_params.items():
-                    params_result += f'{k} = {v}\n'
+                    result += f'{k} = {v}\n'
 
             else:
-                params_result += f"{space}_params = {pformat(space_params, indent=4)}"
+                no_params = HyperoptTools._space_params(non_optimized, space, 5)
-                params_result = params_result.replace("}", "\n}").replace("{", "{\n ")
 
-            params_result = params_result.replace("\n", "\n    ")
+                result += f"{space}_params = {HyperoptTools._pprint(space_params, no_params)}"
-            print(params_result)
+
+            result = result.replace("\n", "\n    ")
+            print(result)
 
     @staticmethod
     def _space_params(params, space: str, r: int = None) -> Dict:
-        d = params[space]
+        d = params.get(space)
+        if d:
             # Round floats to `r` digits after the decimal point if requested
             return round_dict(d, r) if r else d
+        return {}
+
+    @staticmethod
+    def _pprint(params, non_optimized, indent: int = 4):
+        """
+        Pretty-print hyperopt results (based on 2 dicts - with add. comment)
+        """
+        p = params.copy()
+        p.update(non_optimized)
+        result = '{\n'
+
+        for k, param in p.items():
+            result += " " * indent + f'"{k}": '
+            result += f'"{param}",' if isinstance(param, str) else f'{param},'
+            if k in non_optimized:
+                result += "  # value loaded from strategy"
+            result += "\n"
+        result += '}'
+        return result
 
     @staticmethod
     def is_best_loss(results, current_best_loss: float) -> bool:
-        return results['loss'] < current_best_loss
+        return bool(results['loss'] < current_best_loss)
+
+    @staticmethod
+    def format_results_explanation_string(results_metrics: Dict, stake_currency: str) -> str:
+        """
+        Return the formatted results explanation in a string
+        """
+        return (f"{results_metrics['total_trades']:6d} trades. "
+                f"{results_metrics['wins']}/{results_metrics['draws']}"
+                f"/{results_metrics['losses']} Wins/Draws/Losses. "
+                f"Avg profit {results_metrics['profit_mean'] * 100: 6.2f}%. "
+                f"Median profit {results_metrics['profit_median'] * 100: 6.2f}%. "
+                f"Total profit {results_metrics['profit_total_abs']: 11.8f} {stake_currency} "
+                f"({results_metrics['profit_total'] * 100: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). "
+                f"Avg duration {results_metrics['holding_avg']} min."
+                ).encode(locale.getpreferredencoding(), 'replace').decode('utf-8')
 
     @staticmethod
     def _format_explanation_string(results, total_epochs) -> str:
@@ -156,12 +222,27 @@ class HyperoptTools():
         if 'results_metrics.winsdrawslosses' not in trials.columns:
             # Ensure compatibility with older versions of hyperopt results
             trials['results_metrics.winsdrawslosses'] = 'N/A'
+        legacy_mode = True
 
+        if 'results_metrics.total_trades' in trials:
+            legacy_mode = False
+            # New mode, using backtest result for metrics
+            trials['results_metrics.winsdrawslosses'] = trials.apply(
+                lambda x: f"{x['results_metrics.wins']} {x['results_metrics.draws']:>4} "
+                          f"{x['results_metrics.losses']:>4}", axis=1)
+            trials = trials[['Best', 'current_epoch', 'results_metrics.total_trades',
+                             'results_metrics.winsdrawslosses',
+                             'results_metrics.profit_mean', 'results_metrics.profit_total_abs',
+                             'results_metrics.profit_total', 'results_metrics.holding_avg',
+                             'loss', 'is_initial_point', 'is_best']]
+        else:
+            # Legacy mode
             trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
                              'results_metrics.winsdrawslosses',
                              'results_metrics.avg_profit', 'results_metrics.total_profit',
                              'results_metrics.profit', 'results_metrics.duration',
                              'loss', 'is_initial_point', 'is_best']]
+
         trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
                           'Total profit', 'Profit', 'Avg duration', 'Objective',
                           'is_initial_point', 'is_best']
@@ -171,26 +252,28 @@ class HyperoptTools():
         trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best'
         trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
         trials['Trades'] = trials['Trades'].astype(str)
-
+        perc_multi = 1 if legacy_mode else 100
         trials['Epoch'] = trials['Epoch'].apply(
             lambda x: '{}/{}'.format(str(x).rjust(len(str(total_epochs)), ' '), total_epochs)
         )
         trials['Avg profit'] = trials['Avg profit'].apply(
-            lambda x: '{:,.2f}%'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
+            lambda x: f'{x * perc_multi:,.2f}%'.rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
         )
         trials['Avg duration'] = trials['Avg duration'].apply(
-            lambda x: '{:,.1f} m'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
+            lambda x: f'{x:,.1f} m'.rjust(7, ' ') if isinstance(x, float) else f"{x}"
+                      if not isna(x) else "--".rjust(7, ' ')
         )
         trials['Objective'] = trials['Objective'].apply(
-            lambda x: '{:,.5f}'.format(x).rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ')
+            lambda x: f'{x:,.5f}'.rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ')
         )
 
+        stake_currency = config['stake_currency']
         trials['Profit'] = trials.apply(
-            lambda x: '{:,.8f} {} {}'.format(
+            lambda x: '{} {}'.format(
-                x['Total profit'], config['stake_currency'],
+                round_coin_value(x['Total profit'], stake_currency),
-                '({:,.2f}%)'.format(x['Profit']).rjust(10, ' ')
+                '({:,.2f}%)'.format(x['Profit'] * perc_multi).rjust(10, ' ')
-            ).rjust(25+len(config['stake_currency']))
+            ).rjust(25+len(stake_currency))
-            if x['Total profit'] != 0.0 else '--'.rjust(25+len(config['stake_currency'])),
+            if x['Total profit'] != 0.0 else '--'.rjust(25+len(stake_currency)),
             axis=1
         )
         trials = trials.drop(columns=['Total profit'])
@@ -251,6 +334,16 @@ class HyperoptTools():
         trials['Best'] = ''
         trials['Stake currency'] = config['stake_currency']
 
+        if 'results_metrics.total_trades' in trials:
+            base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades',
+                            'results_metrics.profit_mean', 'results_metrics.profit_median',
+                            'results_metrics.profit_total',
+                            'Stake currency',
+                            'results_metrics.profit_total_abs', 'results_metrics.holding_avg',
+                            'loss', 'is_initial_point', 'is_best']
+            perc_multi = 100
+        else:
+            perc_multi = 1
             base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count',
                             'results_metrics.avg_profit', 'results_metrics.median_profit',
                             'results_metrics.total_profit',
@@ -272,21 +365,23 @@ class HyperoptTools():
         trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
         trials['Epoch'] = trials['Epoch'].astype(str)
         trials['Trades'] = trials['Trades'].astype(str)
+        trials['Median profit'] = trials['Median profit'] * perc_multi
 
         trials['Total profit'] = trials['Total profit'].apply(
-            lambda x: '{:,.8f}'.format(x) if x != 0.0 else ""
+            lambda x: f'{x:,.8f}' if x != 0.0 else ""
         )
         trials['Profit'] = trials['Profit'].apply(
-            lambda x: '{:,.2f}'.format(x) if not isna(x) else ""
+            lambda x: f'{x:,.2f}' if not isna(x) else ""
         )
         trials['Avg profit'] = trials['Avg profit'].apply(
-            lambda x: '{:,.2f}%'.format(x) if not isna(x) else ""
+            lambda x: f'{x * perc_multi:,.2f}%' if not isna(x) else ""
         )
         trials['Avg duration'] = trials['Avg duration'].apply(
-            lambda x: '{:,.1f} m'.format(x) if not isna(x) else ""
+            lambda x: f'{x:,.1f} m' if isinstance(
+                x, float) else f"{x.total_seconds() // 60:,.1f} m" if not isna(x) else ""
         )
         trials['Objective'] = trials['Objective'].apply(
-            lambda x: '{:,.5f}'.format(x) if x != 100000 else ""
+            lambda x: f'{x:,.5f}' if x != 100000 else ""
         )
 
         trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit'])

freqtrade/optimize/optimize_reports.py

@@ -3,7 +3,6 @@ from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any, Dict, List, Union
 
-from arrow import Arrow
 from numpy import int64
 from pandas import DataFrame
 from tabulate import tabulate
@@ -44,7 +43,7 @@ def _get_line_floatfmt(stake_currency: str) -> List[str]:
     Generate floatformat (goes in line with _generate_result_line())
     """
     return ['s', 'd', '.2f', '.2f', f'.{decimals_per_coin(stake_currency)}f',
-            '.2f', 'd', 'd', 'd', 'd']
+            '.2f', 'd', 's', 's']
 
 
 def _get_line_header(first_column: str, stake_currency: str) -> List[str]:
@@ -53,7 +52,17 @@ def _get_line_header(first_column: str, stake_currency: str) -> List[str]:
     """
     return [first_column, 'Buys', 'Avg Profit %', 'Cum Profit %',
             f'Tot Profit {stake_currency}', 'Tot Profit %', 'Avg Duration',
-            'Wins', 'Draws', 'Losses']
+            'Win  Draw  Loss  Win%']
+
+
+def _generate_wins_draws_losses(wins, draws, losses):
+    if wins > 0 and losses == 0:
+        wl_ratio = '100'
+    elif wins == 0:
+        wl_ratio = '0'
+    else:
+        wl_ratio = f'{100.0 / (wins + draws + losses) * wins:.1f}' if losses > 0 else '100'
+    return f'{wins:>4}  {draws:>4}  {losses:>4}  {wl_ratio:>4}'
 
 
 def _generate_result_line(result: DataFrame, starting_balance: int, first_column: str) -> Dict:
@@ -153,7 +162,7 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
     return tabular_data
 
 
-def generate_strategy_metrics(all_results: Dict) -> List[Dict]:
+def generate_strategy_comparison(all_results: Dict) -> List[Dict]:
     """
     Generate summary per strategy
     :param all_results: Dict of <Strategyname: DataFrame> containing results for all strategies
@@ -165,6 +174,17 @@ def generate_strategy_metrics(all_results: Dict) -> List[Dict]:
         tabular_data.append(_generate_result_line(
             results['results'], results['config']['dry_run_wallet'], strategy)
             )
+        try:
+            max_drawdown_per, _, _, _, _ = calculate_max_drawdown(results['results'],
+                                                                  value_col='profit_ratio')
+            max_drawdown_abs, _, _, _, _ = calculate_max_drawdown(results['results'],
+                                                                  value_col='profit_abs')
+        except ValueError:
+            max_drawdown_per = 0
+            max_drawdown_abs = 0
+        tabular_data[-1]['max_drawdown_per'] = round(max_drawdown_per * 100, 2)
+        tabular_data[-1]['max_drawdown_abs'] = \
+            round_coin_value(max_drawdown_abs, results['config']['stake_currency'], False)
     return tabular_data
 
 
@@ -194,7 +214,40 @@ def generate_edge_table(results: dict) -> str:
                     floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")  # type: ignore
 
 
+def generate_trading_stats(results: DataFrame) -> Dict[str, Any]:
+    """ Generate overall trade statistics """
+    if len(results) == 0:
+        return {
+            'wins': 0,
+            'losses': 0,
+            'draws': 0,
+            'holding_avg': timedelta(),
+            'winner_holding_avg': timedelta(),
+            'loser_holding_avg': timedelta(),
+        }
+
+    winning_trades = results.loc[results['profit_ratio'] > 0]
+    draw_trades = results.loc[results['profit_ratio'] == 0]
+    losing_trades = results.loc[results['profit_ratio'] < 0]
+    zero_duration_trades = len(results.loc[(results['trade_duration'] == 0) &
+                                           (results['sell_reason'] == 'trailing_stop_loss')])
+
+    return {
+        'wins': len(winning_trades),
+        'losses': len(losing_trades),
+        'draws': len(draw_trades),
+        'holding_avg': (timedelta(minutes=round(results['trade_duration'].mean()))
+                        if not results.empty else timedelta()),
+        'winner_holding_avg': (timedelta(minutes=round(winning_trades['trade_duration'].mean()))
+                               if not winning_trades.empty else timedelta()),
+        'loser_holding_avg': (timedelta(minutes=round(losing_trades['trade_duration'].mean()))
+                              if not losing_trades.empty else timedelta()),
+        'zero_duration_trades': zero_duration_trades,
+    }
+
+
 def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
+    """ Generate daily statistics """
     if len(results) == 0:
         return {
             'backtest_best_day': 0,
@@ -204,8 +257,6 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
             'winning_days': 0,
             'draw_days': 0,
             'losing_days': 0,
-            'winner_holding_avg': timedelta(),
-            'loser_holding_avg': timedelta(),
         }
     daily_profit_rel = results.resample('1d', on='close_date')['profit_ratio'].sum()
     daily_profit = results.resample('1d', on='close_date')['profit_abs'].sum().round(10)
@@ -217,9 +268,6 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
     draw_days = sum(daily_profit == 0)
     losing_days = sum(daily_profit < 0)
 
-    winning_trades = results.loc[results['profit_ratio'] > 0]
-    losing_trades = results.loc[results['profit_ratio'] < 0]
-
     return {
         'backtest_best_day': best_rel,
         'backtest_worst_day': worst_rel,
@@ -228,33 +276,28 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
         'winning_days': winning_days,
         'draw_days': draw_days,
         'losing_days': losing_days,
-        'winner_holding_avg': (timedelta(minutes=round(winning_trades['trade_duration'].mean()))
-                               if not winning_trades.empty else timedelta()),
-        'loser_holding_avg': (timedelta(minutes=round(losing_trades['trade_duration'].mean()))
-                              if not losing_trades.empty else timedelta()),
     }
 
 
-def generate_backtest_stats(btdata: Dict[str, DataFrame],
+def generate_strategy_stats(btdata: Dict[str, DataFrame],
-                            all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]],
+                            strategy: str,
-                            min_date: Arrow, max_date: Arrow
+                            content: Dict[str, Any],
+                            min_date: datetime, max_date: datetime,
+                            market_change: float
                             ) -> Dict[str, Any]:
     """
     :param btdata: Backtest data
-    :param all_results: backtest result - dictionary in the form:
+    :param strategy: Strategy name
-                     { Strategy: {'results: results, 'config: config}}.
+    :param content: Backtest result data in the format:
+                    {'results: results, 'config: config}}.
     :param min_date: Backtest start date
     :param max_date: Backtest end date
-    :return:
+    :param market_change: float indicating the market change
-    Dictionary containing results per strategy and a stratgy summary.
+    :return: Dictionary containing results per strategy and a stratgy summary.
     """
-    result: Dict[str, Any] = {'strategy': {}}
-    market_change = calculate_market_change(btdata, 'close')
-
-    for strategy, content in all_results.items():
     results: Dict[str, DataFrame] = content['results']
     if not isinstance(results, DataFrame):
-            continue
+        return {}
     config = content['config']
     max_open_trades = min(config['max_open_trades'], len(btdata.keys()))
     starting_balance = config['dry_run_wallet']
@@ -270,6 +313,7 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
                                               results=results.loc[results['is_open']],
                                               skip_nan=True)
     daily_stats = generate_daily_stats(results)
+    trade_stats = generate_trading_stats(results)
     best_pair = max([pair for pair in pair_results if pair['key'] != 'TOTAL'],
                     key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
     worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'],
@@ -290,12 +334,13 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
         'total_volume': float(results['stake_amount'].sum()),
         'avg_stake_amount': results['stake_amount'].mean() if len(results) > 0 else 0,
         'profit_mean': results['profit_ratio'].mean() if len(results) > 0 else 0,
+        'profit_median': results['profit_ratio'].median() if len(results) > 0 else 0,
         'profit_total': results['profit_abs'].sum() / starting_balance,
         'profit_total_abs': results['profit_abs'].sum(),
-            'backtest_start': min_date.datetime,
+        'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT),
-            'backtest_start_ts': min_date.int_timestamp * 1000,
+        'backtest_start_ts': int(min_date.timestamp() * 1000),
-            'backtest_end': max_date.datetime,
+        'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT),
-            'backtest_end_ts': max_date.int_timestamp * 1000,
+        'backtest_end_ts': int(max_date.timestamp() * 1000),
         'backtest_days': backtest_days,
 
         'backtest_run_start_ts': content['backtest_start_time'],
@@ -310,6 +355,7 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
         'starting_balance': starting_balance,
         'dry_run_wallet': starting_balance,
         'final_balance': content['final_balance'],
+        'rejected_signals': content['rejected_signals'],
         'max_open_trades': max_open_trades,
         'max_open_trades_setting': (config['max_open_trades']
                                     if config['max_open_trades'] != float('inf') else -1),
@@ -330,8 +376,8 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
         'sell_profit_offset': config['ask_strategy']['sell_profit_offset'],
         'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'],
         **daily_stats,
+        **trade_stats
     }
-        result['strategy'][strategy] = strat_stats
 
     try:
         max_drawdown, _, _, _, _ = calculate_max_drawdown(
@@ -341,9 +387,9 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
         strat_stats.update({
             'max_drawdown': max_drawdown,
             'max_drawdown_abs': drawdown_abs,
-                'drawdown_start': drawdown_start,
+            'drawdown_start': drawdown_start.strftime(DATETIME_PRINT_FORMAT),
             'drawdown_start_ts': drawdown_start.timestamp() * 1000,
-                'drawdown_end': drawdown_end,
+            'drawdown_end': drawdown_end.strftime(DATETIME_PRINT_FORMAT),
             'drawdown_end_ts': drawdown_end.timestamp() * 1000,
 
             'max_drawdown_low': low_val,
@@ -370,7 +416,30 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
             'csum_max': 0
         })
 
-    strategy_results = generate_strategy_metrics(all_results=all_results)
+    return strat_stats
+
+
+def generate_backtest_stats(btdata: Dict[str, DataFrame],
+                            all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]],
+                            min_date: datetime, max_date: datetime
+                            ) -> Dict[str, Any]:
+    """
+    :param btdata: Backtest data
+    :param all_results: backtest result - dictionary in the form:
+                     { Strategy: {'results: results, 'config: config}}.
+    :param min_date: Backtest start date
+    :param max_date: Backtest end date
+    :return: Dictionary containing results per strategy and a stratgy summary.
+    """
+    result: Dict[str, Any] = {'strategy': {}}
+    market_change = calculate_market_change(btdata, 'close')
+
+    for strategy, content in all_results.items():
+        strat_stats = generate_strategy_stats(btdata, strategy, content,
+                                              min_date, max_date, market_change=market_change)
+        result['strategy'][strategy] = strat_stats
+
+    strategy_results = generate_strategy_comparison(all_results=all_results)
 
     result['strategy_comparison'] = strategy_results
 
@@ -393,7 +462,8 @@ def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: st
     floatfmt = _get_line_floatfmt(stake_currency)
     output = [[
         t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'],
-        t['profit_total_pct'], t['duration_avg'], t['wins'], t['draws'], t['losses']
+        t['profit_total_pct'], t['duration_avg'],
+        _generate_wins_draws_losses(t['wins'], t['draws'], t['losses'])
     ] for t in pair_results]
     # Ignore type as floatfmt does allow tuples but mypy does not know that
     return tabulate(output, headers=headers,
@@ -410,9 +480,7 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
     headers = [
         'Sell Reason',
         'Sells',
-        'Wins',
+        'Win  Draws  Loss  Win%',
-        'Draws',
-        'Losses',
         'Avg Profit %',
         'Cum Profit %',
         f'Tot Profit {stake_currency}',
@@ -420,7 +488,8 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
     ]
 
     output = [[
-        t['sell_reason'], t['trades'], t['wins'], t['draws'], t['losses'],
+        t['sell_reason'], t['trades'],
+        _generate_wins_draws_losses(t['wins'], t['draws'], t['losses']),
         t['profit_mean_pct'], t['profit_sum_pct'],
         round_coin_value(t['profit_total_abs'], stake_currency, False),
         t['profit_total_pct'],
@@ -438,11 +507,22 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str:
     """
     floatfmt = _get_line_floatfmt(stake_currency)
     headers = _get_line_header('Strategy', stake_currency)
+    # _get_line_header() is also used for per-pair summary. Per-pair drawdown is mostly useless
+    # therefore we slip this column in only for strategy summary here.
+    headers.append('Drawdown')
+
+    # Align drawdown string on the center two space separator.
+    drawdown = [f'{t["max_drawdown_per"]:.2f}' for t in strategy_results]
+    dd_pad_abs = max([len(t['max_drawdown_abs']) for t in strategy_results])
+    dd_pad_per = max([len(dd) for dd in drawdown])
+    drawdown = [f'{t["max_drawdown_abs"]:>{dd_pad_abs}} {stake_currency}  {dd:>{dd_pad_per}}%'
+                for t, dd in zip(strategy_results, drawdown)]
 
     output = [[
         t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'],
-        t['profit_total_pct'], t['duration_avg'], t['wins'], t['draws'], t['losses']
+        t['profit_total_pct'], t['duration_avg'],
-    ] for t in strategy_results]
+        _generate_wins_draws_losses(t['wins'], t['draws'], t['losses']), drawdown]
+        for t, drawdown in zip(strategy_results, drawdown)]
     # Ignore type as floatfmt does allow tuples but mypy does not know that
     return tabulate(output, headers=headers,
                     floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
@@ -452,9 +532,21 @@ def text_table_add_metrics(strat_results: Dict) -> str:
     if len(strat_results['trades']) > 0:
         best_trade = max(strat_results['trades'], key=lambda x: x['profit_ratio'])
         worst_trade = min(strat_results['trades'], key=lambda x: x['profit_ratio'])
+
+        # Newly added fields should be ignored if they are missing in strat_results. hyperopt-show
+        # command stores these results and newer version of freqtrade must be able to handle old
+        # results with missing new fields.
+        zero_duration_trades = '--'
+
+        if 'zero_duration_trades' in strat_results:
+            zero_duration_trades_per = \
+                100.0 / strat_results['total_trades'] * strat_results['zero_duration_trades']
+            zero_duration_trades = f'{zero_duration_trades_per:.2f}% ' \
+                                   f'({strat_results["zero_duration_trades"]})'
+
         metrics = [
-            ('Backtesting from', strat_results['backtest_start'].strftime(DATETIME_PRINT_FORMAT)),
+            ('Backtesting from', strat_results['backtest_start']),
-            ('Backtesting to', strat_results['backtest_end'].strftime(DATETIME_PRINT_FORMAT)),
+            ('Backtesting to', strat_results['backtest_end']),
             ('Max open trades', strat_results['max_open_trades']),
             ('', ''),  # Empty line to improve readability
             ('Total trades', strat_results['total_trades']),
@@ -464,13 +556,12 @@ def text_table_add_metrics(strat_results: Dict) -> str:
                                                strat_results['stake_currency'])),
             ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'],
                                                   strat_results['stake_currency'])),
-            ('Total profit %', f"{round(strat_results['profit_total'] * 100, 2)}%"),
+            ('Total profit %', f"{round(strat_results['profit_total'] * 100, 2):}%"),
             ('Trades per day', strat_results['trades_per_day']),
             ('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'],
                                                    strat_results['stake_currency'])),
             ('Total trade volume', round_coin_value(strat_results['total_volume'],
                                                     strat_results['stake_currency'])),
-
             ('', ''),  # Empty line to improve readability
             ('Best Pair', f"{strat_results['best_pair']['key']} "
                           f"{round(strat_results['best_pair']['profit_sum_pct'], 2)}%"),
@@ -488,6 +579,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
                 f"{strat_results['draw_days']} / {strat_results['losing_days']}"),
             ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"),
             ('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"),
+            ('Zero Duration Trades', zero_duration_trades),
+            ('Rejected Buy signals', strat_results.get('rejected_signals', 'N/A')),
             ('', ''),  # Empty line to improve readability
 
             ('Min balance', round_coin_value(strat_results['csum_min'],
@@ -502,8 +595,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
                                                strat_results['stake_currency'])),
             ('Drawdown low', round_coin_value(strat_results['max_drawdown_low'],
                                               strat_results['stake_currency'])),
-            ('Drawdown Start', strat_results['drawdown_start'].strftime(DATETIME_PRINT_FORMAT)),
+            ('Drawdown Start', strat_results['drawdown_start']),
-            ('Drawdown End', strat_results['drawdown_end'].strftime(DATETIME_PRINT_FORMAT)),
+            ('Drawdown End', strat_results['drawdown_end']),
             ('Market change', f"{round(strat_results['market_change'] * 100, 2)}%"),
         ]
 
@@ -522,11 +615,10 @@ def text_table_add_metrics(strat_results: Dict) -> str:
         return message
 
 
-def show_backtest_results(config: Dict, backtest_stats: Dict):
+def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: str):
-    stake_currency = config['stake_currency']
+    """
-
+    Print results for one strategy
-    for strategy, results in backtest_stats['strategy'].items():
+    """
-
     # Print results
     print(f"Result for strategy {strategy}")
     table = text_table_bt_results(results['results_per_pair'], stake_currency=stake_currency)
@@ -554,6 +646,13 @@ def show_backtest_results(config: Dict, backtest_stats: Dict):
         print('=' * len(table.splitlines()[0]))
     print()
 
+
+def show_backtest_results(config: Dict, backtest_stats: Dict):
+    stake_currency = config['stake_currency']
+
+    for strategy, results in backtest_stats['strategy'].items():
+        show_backtest_result(strategy, results, stake_currency)
+
     if len(backtest_stats['strategy']) > 1:
         # Print Strategy summary table
 

freqtrade/optimize/space/__init__.py

@@ -0,0 +1,4 @@
+# flake8: noqa: F401
+from skopt.space import Categorical, Dimension, Integer, Real
+
+from .decimalspace import SKDecimal

freqtrade/optimize/space/decimalspace.py

@@ -0,0 +1,33 @@
+import numpy as np
+from skopt.space import Integer
+
+
+class SKDecimal(Integer):
+
+    def __init__(self, low, high, decimals=3, prior="uniform", base=10, transform=None,
+                 name=None, dtype=np.int64):
+        self.decimals = decimals
+        _low = int(low * pow(10, self.decimals))
+        _high = int(high * pow(10, self.decimals))
+        # trunc to precision to avoid points out of space
+        self.low_orig = round(_low * pow(0.1, self.decimals), self.decimals)
+        self.high_orig = round(_high * pow(0.1, self.decimals), self.decimals)
+
+        super().__init__(_low, _high, prior, base, transform, name, dtype)
+
+    def __repr__(self):
+        return "Decimal(low={}, high={}, decimals={}, prior='{}', transform='{}')".format(
+            self.low_orig, self.high_orig, self.decimals, self.prior, self.transform_)
+
+    def __contains__(self, point):
+        if isinstance(point, list):
+            point = np.array(point)
+        return self.low_orig <= point <= self.high_orig
+
+    def transform(self, Xt):
+        aa = [int(x * pow(10, self.decimals)) for x in Xt]
+        return super().transform(aa)
+
+    def inverse_transform(self, Xt):
+        res = super().inverse_transform(Xt)
+        return [round(x * pow(0.1, self.decimals), self.decimals) for x in res]

freqtrade/persistence/migrations.py

@@ -123,6 +123,27 @@ def migrate_open_orders_to_trades(engine):
         """)
 
 
+def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, cols: List):
+    # Schema migration necessary
+    engine.execute(f"alter table orders rename to {table_back_name}")
+    # drop indexes on backup table
+    for index in inspector.get_indexes(table_back_name):
+        engine.execute(f"drop index {index['name']}")
+
+    # let SQLAlchemy create the schema as required
+    decl_base.metadata.create_all(engine)
+
+    engine.execute(f"""
+        insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status,
+        symbol, order_type, side, price, amount, filled, average, remaining, cost, order_date,
+        order_filled_date, order_update_date)
+        select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status,
+        symbol, order_type, side, price, amount, filled, null average, remaining, cost, order_date,
+        order_filled_date, order_update_date
+        from {table_back_name}
+        """)
+
+
 def check_migrate(engine, decl_base, previous_tables) -> None:
     """
     Checks if migration is necessary and migrates if necessary
@@ -145,6 +166,11 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
         logger.info('Moving open orders to Orders table.')
         migrate_open_orders_to_trades(engine)
     else:
-        pass
+        cols_order = inspector.get_columns('orders')
+
+        if not has_column(cols_order, 'average'):
+            tabs = get_table_names_for_table(inspector, 'orders')
             # Empty for now - as there is only one iteration of the orders table so far.
-        # table_back_name = get_backup_name(tabs, 'orders_bak')
+            table_back_name = get_backup_name(tabs, 'orders_bak')
+
+            migrate_orders_table(decl_base, inspector, engine, table_back_name, cols)

freqtrade/persistence/models.py

@@ -6,7 +6,6 @@ from datetime import datetime, timezone
 from decimal import Decimal
 from typing import Any, Dict, List, Optional
 
-import arrow
 from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, String,
                         create_engine, desc, func, inspect)
 from sqlalchemy.exc import NoSuchModuleError
@@ -113,16 +112,17 @@ class Order(_DECL_BASE):
 
     trade = relationship("Trade", back_populates="orders")
 
-    ft_order_side = Column(String, nullable=False)
+    ft_order_side = Column(String(25), nullable=False)
-    ft_pair = Column(String, nullable=False)
+    ft_pair = Column(String(25), nullable=False)
     ft_is_open = Column(Boolean, nullable=False, default=True, index=True)
 
-    order_id = Column(String, nullable=False, index=True)
+    order_id = Column(String(255), nullable=False, index=True)
-    status = Column(String, nullable=True)
+    status = Column(String(255), nullable=True)
-    symbol = Column(String, nullable=True)
+    symbol = Column(String(25), nullable=True)
-    order_type = Column(String, nullable=True)
+    order_type = Column(String(50), nullable=True)
-    side = Column(String, nullable=True)
+    side = Column(String(25), nullable=True)
     price = Column(Float, nullable=True)
+    average = Column(Float, nullable=True)
     amount = Column(Float, nullable=True)
     filled = Column(Float, nullable=True)
     remaining = Column(Float, nullable=True)
@@ -151,6 +151,7 @@ class Order(_DECL_BASE):
         self.price = order.get('price', self.price)
         self.amount = order.get('amount', self.amount)
         self.filled = order.get('filled', self.filled)
+        self.average = order.get('average', self.average)
         self.remaining = order.get('remaining', self.remaining)
         self.cost = order.get('cost', self.cost)
         if 'timestamp' in order and order['timestamp'] is not None:
@@ -160,8 +161,8 @@ class Order(_DECL_BASE):
         if self.status in ('closed', 'canceled', 'cancelled'):
             self.ft_is_open = False
             if order.get('filled', 0) > 0:
-                self.order_filled_date = arrow.utcnow().datetime
+                self.order_filled_date = datetime.now(timezone.utc)
-        self.order_update_date = arrow.utcnow().datetime
+        self.order_update_date = datetime.now(timezone.utc)
 
     @staticmethod
     def update_orders(orders: List['Order'], order: Dict[str, Any]):
@@ -294,15 +295,12 @@ class LocalTrade():
             'fee_close_cost': self.fee_close_cost,
             'fee_close_currency': self.fee_close_currency,
 
-            'open_date_hum': arrow.get(self.open_date).humanize(),
             'open_date': self.open_date.strftime(DATETIME_PRINT_FORMAT),
             'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000),
             'open_rate': self.open_rate,
             'open_rate_requested': self.open_rate_requested,
             'open_trade_value': round(self.open_trade_value, 8),
 
-            'close_date_hum': (arrow.get(self.close_date).humanize()
-                               if self.close_date else None),
             'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT)
                            if self.close_date else None),
             'close_timestamp': int(self.close_date.replace(
@@ -551,6 +549,8 @@ class LocalTrade():
             rate=(rate or self.close_rate),
             fee=(fee or self.fee_close)
         )
+        if self.open_trade_value == 0.0:
+            return 0.0
         profit_ratio = (close_trade_value / self.open_trade_value) - 1
         return float(f"{profit_ratio:.8f}")
 
@@ -569,23 +569,6 @@ class LocalTrade():
         else:
             return None
 
-    @staticmethod
-    def get_trades(trade_filter=None) -> Query:
-        """
-        Helper function to query Trades using filters.
-        :param trade_filter: Optional filter to apply to trades
-                             Can be either a Filter object, or a List of filters
-                             e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])`
-                             e.g. `(trade_filter=Trade.id == trade_id)`
-        :return: unsorted query object
-        """
-        if trade_filter is not None:
-            if not isinstance(trade_filter, list):
-                trade_filter = [trade_filter]
-            return Trade.query.filter(*trade_filter)
-        else:
-            return Trade.query
-
     @staticmethod
     def get_trades_proxy(*, pair: str = None, is_open: bool = None,
                          open_date: datetime = None, close_date: datetime = None,
@@ -638,83 +621,7 @@ class LocalTrade():
         """
         Query trades from persistence layer
         """
-        return Trade.get_trades(Trade.is_open.is_(True)).all()
+        return Trade.get_trades_proxy(is_open=True)
-
-    @staticmethod
-    def get_open_order_trades():
-        """
-        Returns all open trades
-        """
-        return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
-
-    @staticmethod
-    def get_open_trades_without_assigned_fees():
-        """
-        Returns all open trades which don't have open fees set correctly
-        """
-        return Trade.get_trades([Trade.fee_open_currency.is_(None),
-                                 Trade.orders.any(),
-                                 Trade.is_open.is_(True),
-                                 ]).all()
-
-    @staticmethod
-    def get_sold_trades_without_assigned_fees():
-        """
-        Returns all closed trades which don't have fees set correctly
-        """
-        return Trade.get_trades([Trade.fee_close_currency.is_(None),
-                                 Trade.orders.any(),
-                                 Trade.is_open.is_(False),
-                                 ]).all()
-
-    @staticmethod
-    def total_open_trades_stakes() -> float:
-        """
-        Calculates total invested amount in open trades
-        in stake currency
-        """
-        if Trade.use_db:
-            total_open_stake_amount = Trade.query.with_entities(
-                func.sum(Trade.stake_amount)).filter(Trade.is_open.is_(True)).scalar()
-        else:
-            total_open_stake_amount = sum(
-                t.stake_amount for t in Trade.get_trades_proxy(is_open=True))
-        return total_open_stake_amount or 0
-
-    @staticmethod
-    def get_overall_performance() -> List[Dict[str, Any]]:
-        """
-        Returns List of dicts containing all Trades, including profit and trade count
-        """
-        pair_rates = Trade.query.with_entities(
-            Trade.pair,
-            func.sum(Trade.close_profit).label('profit_sum'),
-            func.count(Trade.pair).label('count')
-        ).filter(Trade.is_open.is_(False))\
-            .group_by(Trade.pair) \
-            .order_by(desc('profit_sum')) \
-            .all()
-        return [
-            {
-                'pair': pair,
-                'profit': rate,
-                'count': count
-            }
-            for pair, rate, count in pair_rates
-        ]
-
-    @staticmethod
-    def get_best_pair():
-        """
-        Get best pair with closed trade.
-        :returns: Tuple containing (pair, profit_sum)
-        """
-        best_pair = Trade.query.with_entities(
-            Trade.pair, func.sum(Trade.close_profit).label('profit_sum')
-        ).filter(Trade.is_open.is_(False)) \
-            .group_by(Trade.pair) \
-            .order_by(desc('profit_sum')).first()
-        return best_pair
 
     @staticmethod
     def stoploss_reinitialization(desired_stoploss):
@@ -751,15 +658,15 @@ class Trade(_DECL_BASE, LocalTrade):
 
     orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan")
 
-    exchange = Column(String, nullable=False)
+    exchange = Column(String(25), nullable=False)
-    pair = Column(String, nullable=False, index=True)
+    pair = Column(String(25), nullable=False, index=True)
     is_open = Column(Boolean, nullable=False, default=True, index=True)
     fee_open = Column(Float, nullable=False, default=0.0)
     fee_open_cost = Column(Float, nullable=True)
-    fee_open_currency = Column(String, nullable=True)
+    fee_open_currency = Column(String(25), nullable=True)
     fee_close = Column(Float, nullable=False, default=0.0)
     fee_close_cost = Column(Float, nullable=True)
-    fee_close_currency = Column(String, nullable=True)
+    fee_close_currency = Column(String(25), nullable=True)
     open_rate = Column(Float)
     open_rate_requested = Column(Float)
     # open_trade_value - calculated via _calc_open_trade_value
@@ -773,7 +680,7 @@ class Trade(_DECL_BASE, LocalTrade):
     amount_requested = Column(Float)
     open_date = Column(DateTime, nullable=False, default=datetime.utcnow)
     close_date = Column(DateTime)
-    open_order_id = Column(String)
+    open_order_id = Column(String(255))
     # absolute value of the stop loss
     stop_loss = Column(Float, nullable=True, default=0.0)
     # percentage value of the stop loss
@@ -783,16 +690,16 @@ class Trade(_DECL_BASE, LocalTrade):
     # percentage value of the initial stop loss
     initial_stop_loss_pct = Column(Float, nullable=True)
     # stoploss order id which is on exchange
-    stoploss_order_id = Column(String, nullable=True, index=True)
+    stoploss_order_id = Column(String(255), nullable=True, index=True)
     # last update time of the stoploss order on exchange
     stoploss_last_update = Column(DateTime, nullable=True)
     # absolute value of the highest reached price
     max_rate = Column(Float, nullable=True, default=0.0)
     # Lowest price reached
     min_rate = Column(Float, nullable=True)
-    sell_reason = Column(String, nullable=True)
+    sell_reason = Column(String(100), nullable=True)
-    sell_order_status = Column(String, nullable=True)
+    sell_order_status = Column(String(100), nullable=True)
-    strategy = Column(String, nullable=True)
+    strategy = Column(String(100), nullable=True)
     timeframe = Column(Integer, nullable=True)
 
     def __init__(self, **kwargs):
@@ -812,7 +719,7 @@ class Trade(_DECL_BASE, LocalTrade):
                          open_date: datetime = None, close_date: datetime = None,
                          ) -> List['LocalTrade']:
         """
-        Helper function to query Trades.
+        Helper function to query Trades.j
         Returns a List of trades, filtered on the parameters given.
         In live mode, converts the filter to a database query and returns all rows
         In Backtest mode, uses filters on Trade.trades to get the result.
@@ -837,6 +744,109 @@ class Trade(_DECL_BASE, LocalTrade):
                 close_date=close_date
             )
 
+    @staticmethod
+    def get_trades(trade_filter=None) -> Query:
+        """
+        Helper function to query Trades using filters.
+        NOTE: Not supported in Backtesting.
+        :param trade_filter: Optional filter to apply to trades
+                             Can be either a Filter object, or a List of filters
+                             e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])`
+                             e.g. `(trade_filter=Trade.id == trade_id)`
+        :return: unsorted query object
+        """
+        if not Trade.use_db:
+            raise NotImplementedError('`Trade.get_trades()` not supported in backtesting mode.')
+        if trade_filter is not None:
+            if not isinstance(trade_filter, list):
+                trade_filter = [trade_filter]
+            return Trade.query.filter(*trade_filter)
+        else:
+            return Trade.query
+
+    @staticmethod
+    def get_open_order_trades():
+        """
+        Returns all open trades
+        NOTE: Not supported in Backtesting.
+        """
+        return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
+
+    @staticmethod
+    def get_open_trades_without_assigned_fees():
+        """
+        Returns all open trades which don't have open fees set correctly
+        NOTE: Not supported in Backtesting.
+        """
+        return Trade.get_trades([Trade.fee_open_currency.is_(None),
+                                 Trade.orders.any(),
+                                 Trade.is_open.is_(True),
+                                 ]).all()
+
+    @staticmethod
+    def get_sold_trades_without_assigned_fees():
+        """
+        Returns all closed trades which don't have fees set correctly
+        NOTE: Not supported in Backtesting.
+        """
+        return Trade.get_trades([Trade.fee_close_currency.is_(None),
+                                 Trade.orders.any(),
+                                 Trade.is_open.is_(False),
+                                 ]).all()
+
+    @staticmethod
+    def total_open_trades_stakes() -> float:
+        """
+        Calculates total invested amount in open trades
+        in stake currency
+        """
+        if Trade.use_db:
+            total_open_stake_amount = Trade.query.with_entities(
+                func.sum(Trade.stake_amount)).filter(Trade.is_open.is_(True)).scalar()
+        else:
+            total_open_stake_amount = sum(
+                t.stake_amount for t in LocalTrade.get_trades_proxy(is_open=True))
+        return total_open_stake_amount or 0
+
+    @staticmethod
+    def get_overall_performance() -> List[Dict[str, Any]]:
+        """
+        Returns List of dicts containing all Trades, including profit and trade count
+        NOTE: Not supported in Backtesting.
+        """
+        pair_rates = Trade.query.with_entities(
+            Trade.pair,
+            func.sum(Trade.close_profit).label('profit_sum'),
+            func.sum(Trade.close_profit_abs).label('profit_sum_abs'),
+            func.count(Trade.pair).label('count')
+        ).filter(Trade.is_open.is_(False))\
+            .group_by(Trade.pair) \
+            .order_by(desc('profit_sum_abs')) \
+            .all()
+        return [
+            {
+                'pair': pair,
+                'profit': profit,
+                'profit_abs': profit_abs,
+                'count': count
+            }
+            for pair, profit, profit_abs, count in pair_rates
+        ]
+
+    @staticmethod
+    def get_best_pair():
+        """
+        Get best pair with closed trade.
+        NOTE: Not supported in Backtesting.
+        :returns: Tuple containing (pair, profit_sum)
+        """
+        best_pair = Trade.query.with_entities(
+            Trade.pair, func.sum(Trade.close_profit).label('profit_sum')
+        ).filter(Trade.is_open.is_(False)) \
+            .group_by(Trade.pair) \
+            .order_by(desc('profit_sum')).first()
+        return best_pair
+
 
 class PairLock(_DECL_BASE):
     """
@@ -846,8 +856,8 @@ class PairLock(_DECL_BASE):
 
     id = Column(Integer, primary_key=True)
 
-    pair = Column(String, nullable=False, index=True)
+    pair = Column(String(25), nullable=False, index=True)
-    reason = Column(String, nullable=True)
+    reason = Column(String(255), nullable=True)
     # Time the pair was locked (start time)
     lock_time = Column(DateTime, nullable=False)
     # Time until the pair is locked (end time)

freqtrade/plot/plotting.py

@@ -77,6 +77,7 @@ def init_plotscript(config, markets: List, startup_candles: int = 0):
         )
     except ValueError as e:
         raise OperationalException(e) from e
+    if not trades.empty:
         trades = trim_dataframe(trades, timerange, 'open_date')
 
     return {"ohlcv": data,
@@ -441,7 +442,7 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra
 
 
 def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
-                          trades: pd.DataFrame, timeframe: str) -> go.Figure:
+                          trades: pd.DataFrame, timeframe: str, stake_currency: str) -> go.Figure:
     # Combine close-values for all pairs, rename columns to "pair"
     df_comb = combine_dataframes_with_mean(data, "close")
 
@@ -466,8 +467,8 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
                         subplot_titles=["AVG Close Price", "Combined Profit", "Profit per pair"])
     fig['layout'].update(title="Freqtrade Profit plot")
     fig['layout']['yaxis1'].update(title='Price')
-    fig['layout']['yaxis2'].update(title='Profit')
+    fig['layout']['yaxis2'].update(title=f'Profit {stake_currency}')
-    fig['layout']['yaxis3'].update(title='Profit')
+    fig['layout']['yaxis3'].update(title=f'Profit {stake_currency}')
     fig['layout']['xaxis']['rangeslider'].update(visible=False)
 
     fig.add_trace(avgclose, 1, 1)
@@ -540,8 +541,11 @@ def load_and_plot_trades(config: Dict[str, Any]):
 
         df_analyzed = strategy.analyze_ticker(data, {'pair': pair})
         df_analyzed = trim_dataframe(df_analyzed, timerange)
+        if not trades.empty:
             trades_pair = trades.loc[trades['pair'] == pair]
             trades_pair = extract_trades_of_period(df_analyzed, trades_pair)
+        else:
+            trades_pair = trades
 
         fig = generate_candlestick_graph(
             pair=pair,
@@ -581,6 +585,7 @@ def plot_profit(config: Dict[str, Any]) -> None:
     # Create an average close price of all the pairs that were involved.
     # this could be useful to gauge the overall market trend
     fig = generate_profit_graph(plot_elements['pairs'], plot_elements['ohlcv'],
-                                trades, config.get('timeframe', '5m'))
+                                trades, config.get('timeframe', '5m'),
+                                config.get('stake_currency', ''))
     store_plot_file(fig, filename='freqtrade-profit-plot.html',
                     directory=config['user_data_dir'] / 'plot', auto_open=True)

freqtrade/plugins/pairlist/AgeFilter.py

@@ -71,14 +71,14 @@ class AgeFilter(IPairList):
                 daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None
                 if not self._validate_pair_loc(p, daily_candles):
                     pairlist.remove(p)
-        logger.info(f"Validated {len(pairlist)} pairs.")
+        self.log_once(f"Validated {len(pairlist)} pairs.", logger.info)
         return pairlist
 
     def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool:
         """
         Validate age for the ticker
         :param pair: Pair that's currently validated
-        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param ticker: ticker dict as returned from ccxt.fetch_tickers()
         :return: True if the pair can stay, false if it should be removed
         """
         # Check symbol in cache
@@ -86,7 +86,7 @@ class AgeFilter(IPairList):
             return True
 
         if daily_candles is not None:
-            if len(daily_candles) > self._min_days_listed:
+            if len(daily_candles) >= self._min_days_listed:
                 # We have fetched at least the minimum required number of daily candles
                 # Add to cache, store the time we last checked this symbol
                 self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000

freqtrade/plugins/pairlist/IPairList.py

@@ -7,7 +7,7 @@ from copy import deepcopy
 from typing import Any, Dict, List
 
 from freqtrade.exceptions import OperationalException
-from freqtrade.exchange import market_is_active
+from freqtrade.exchange import Exchange, market_is_active
 from freqtrade.mixins import LoggingMixin
 
 
@@ -16,7 +16,7 @@ logger = logging.getLogger(__name__)
 
 class IPairList(LoggingMixin, ABC):
 
-    def __init__(self, exchange, pairlistmanager,
+    def __init__(self, exchange: Exchange, pairlistmanager,
                  config: Dict[str, Any], pairlistconfig: Dict[str, Any],
                  pairlist_pos: int) -> None:
         """
@@ -28,7 +28,7 @@ class IPairList(LoggingMixin, ABC):
         """
         self._enabled = True
 
-        self._exchange = exchange
+        self._exchange: Exchange = exchange
         self._pairlistmanager = pairlistmanager
         self._config = config
         self._pairlistconfig = pairlistconfig
@@ -68,12 +68,12 @@ class IPairList(LoggingMixin, ABC):
         filter_pairlist() method.
 
         :param pair: Pair that's currently validated
-        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param ticker: ticker dict as returned from ccxt.fetch_tickers()
         :return: True if the pair can stay, false if it should be removed
         """
         raise NotImplementedError()
 
-    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+    def gen_pairlist(self, tickers: Dict) -> List[str]:
         """
         Generate the pairlist.
 
@@ -84,8 +84,7 @@ class IPairList(LoggingMixin, ABC):
         it will raise the exception if a Pairlist Handler is used at the first
         position in the chain.
 
-        :param cached_pairlist: Previously generated pairlist (cached)
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
-        :param tickers: Tickers (from exchange.get_tickers()).
         :return: List of pairs
         """
         raise OperationalException("This Pairlist Handler should not be used "

freqtrade/plugins/pairlist/PerformanceFilter.py

@@ -39,7 +39,12 @@ class PerformanceFilter(IPairList):
         :return: new allowlist
         """
         # Get the trading performance for pairs from database
+        try:
             performance = pd.DataFrame(Trade.get_overall_performance())
+        except AttributeError:
+            # Performancefilter does not work in backtesting.
+            self.log_once("PerformanceFilter is not available in this mode.", logger.warning)
+            return pairlist
 
         # Skip performance-based sorting if no performance data is available
         if len(performance) == 0:

freqtrade/plugins/pairlist/PrecisionFilter.py

@@ -48,7 +48,7 @@ class PrecisionFilter(IPairList):
         Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very
         low value pairs.
         :param pair: Pair that's currently validated
-        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param ticker: ticker dict as returned from ccxt.fetch_tickers()
         :return: True if the pair can stay, false if it should be removed
         """
         stop_price = ticker['ask'] * self._stoploss

freqtrade/plugins/pairlist/PriceFilter.py

@@ -27,9 +27,13 @@ class PriceFilter(IPairList):
         self._max_price = pairlistconfig.get('max_price', 0)
         if self._max_price < 0:
             raise OperationalException("PriceFilter requires max_price to be >= 0")
+        self._max_value = pairlistconfig.get('max_value', 0)
+        if self._max_value < 0:
+            raise OperationalException("PriceFilter requires max_value to be >= 0")
         self._enabled = ((self._low_price_ratio > 0) or
                          (self._min_price > 0) or
-                         (self._max_price > 0))
+                         (self._max_price > 0) or
+                         (self._max_value > 0))
 
     @property
     def needstickers(self) -> bool:
@@ -51,6 +55,8 @@ class PriceFilter(IPairList):
             active_price_filters.append(f"below {self._min_price:.8f}")
         if self._max_price != 0:
             active_price_filters.append(f"above {self._max_price:.8f}")
+        if self._max_value != 0:
+            active_price_filters.append(f"Value above {self._max_value:.8f}")
 
         if len(active_price_filters):
             return f"{self.name} - Filtering pairs priced {' or '.join(active_price_filters)}."
@@ -61,7 +67,7 @@ class PriceFilter(IPairList):
         """
         Check if if one price-step (pip) is > than a certain barrier.
         :param pair: Pair that's currently validated
-        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param ticker: ticker dict as returned from ccxt.fetch_tickers()
         :return: True if the pair can stay, false if it should be removed
         """
         if ticker.get('last', None) is None or ticker.get('last') == 0:
@@ -79,6 +85,32 @@ class PriceFilter(IPairList):
                               f"because 1 unit is {changeperc * 100:.3f}%", logger.info)
                 return False
 
+        # Perform low_amount check
+        if self._max_value != 0:
+            price = ticker['last']
+            market = self._exchange.markets[pair]
+            limits = market['limits']
+            if ('amount' in limits and 'min' in limits['amount']
+                    and limits['amount']['min'] is not None):
+                min_amount = limits['amount']['min']
+                min_precision = market['precision']['amount']
+
+                min_value = min_amount * price
+                if self._exchange.precisionMode == 4:
+                    # tick size
+                    next_value = (min_amount + min_precision) * price
+                else:
+                    # Decimal places
+                    min_precision = pow(0.1, min_precision)
+                    next_value = (min_amount + min_precision) * price
+                diff = next_value - min_value
+
+                if diff > self._max_value:
+                    self.log_once(f"Removed {pair} from whitelist, "
+                                  f"because min value change of {diff} > {self._max_value}.",
+                                  logger.info)
+                    return False
+
         # Perform min_price check.
         if self._min_price != 0:
             if ticker['last'] < self._min_price:
@@ -89,7 +121,7 @@ class PriceFilter(IPairList):
         # Perform max_price check.
         if self._max_price != 0:
             if ticker['last'] > self._max_price:
-                self.log_once(f"Removed {ticker['symbol']} from whitelist, "
+                self.log_once(f"Removed {pair} from whitelist, "
                               f"because last price > {self._max_price:.8f}", logger.info)
                 return False
 

freqtrade/plugins/pairlist/SpreadFilter.py

@@ -40,7 +40,7 @@ class SpreadFilter(IPairList):
         """
         Validate spread for the ticker
         :param pair: Pair that's currently validated
-        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param ticker: ticker dict as returned from ccxt.fetch_tickers()
         :return: True if the pair can stay, false if it should be removed
         """
         if 'bid' in ticker and 'ask' in ticker and ticker['ask']:

freqtrade/plugins/pairlist/StaticPairList.py

@@ -42,11 +42,10 @@ class StaticPairList(IPairList):
         """
         return f"{self.name}"
 
-    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+    def gen_pairlist(self, tickers: Dict) -> List[str]:
         """
         Generate the pairlist
-        :param cached_pairlist: Previously generated pairlist (cached)
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
-        :param tickers: Tickers (from exchange.get_tickers()).
         :return: List of pairs
         """
         if self._allow_inactive:

freqtrade/plugins/pairlist/VolatilityFilter.py

@@ -90,7 +90,7 @@ class VolatilityFilter(IPairList):
         """
         Validate trading range
         :param pair: Pair that's currently validated
-        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param ticker: ticker dict as returned from ccxt.fetch_tickers()
         :return: True if the pair can stay, false if it should be removed
         """
         # Check symbol in cache

freqtrade/plugins/pairlist/VolumePairList.py

@@ -4,9 +4,10 @@ Volume PairList provider
 Provides dynamic pair list based on trade volumes
 """
 import logging
-from datetime import datetime
 from typing import Any, Dict, List
 
+from cachetools.ttl import TTLCache
+
 from freqtrade.exceptions import OperationalException
 from freqtrade.plugins.pairlist.IPairList import IPairList
 
@@ -33,7 +34,8 @@ class VolumePairList(IPairList):
         self._number_pairs = self._pairlistconfig['number_assets']
         self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume')
         self._min_value = self._pairlistconfig.get('min_value', 0)
-        self.refresh_period = self._pairlistconfig.get('refresh_period', 1800)
+        self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
+        self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
 
         if not self._exchange.exchange_has('fetchTickers'):
             raise OperationalException(
@@ -63,17 +65,19 @@ class VolumePairList(IPairList):
         """
         return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs."
 
-    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+    def gen_pairlist(self, tickers: Dict) -> List[str]:
         """
         Generate the pairlist
-        :param cached_pairlist: Previously generated pairlist (cached)
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
-        :param tickers: Tickers (from exchange.get_tickers()).
         :return: List of pairs
         """
         # Generate dynamic whitelist
         # Must always run if this pairlist is not the first in the list.
-        if self._last_refresh + self.refresh_period < datetime.now().timestamp():
+        pairlist = self._pair_cache.get('pairlist')
-            self._last_refresh = int(datetime.now().timestamp())
+        if pairlist:
+            # Item found - no refresh necessary
+            return pairlist
+        else:
 
             # Use fresh pairlist
             # Check if pair quote currency equals to the stake currency.
@@ -82,9 +86,9 @@ class VolumePairList(IPairList):
                     if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
                         and v[self._sort_key] is not None)]
             pairlist = [s['symbol'] for s in filtered_tickers]
-        else:
+
-            # Use the cached pairlist if it's not time yet to refresh
+            pairlist = self.filter_pairlist(pairlist, tickers)
-            pairlist = cached_pairlist
+            self._pair_cache['pairlist'] = pairlist
 
         return pairlist
 

freqtrade/plugins/pairlist/rangestabilityfilter.py

@@ -83,7 +83,7 @@ class RangeStabilityFilter(IPairList):
         """
         Validate trading range
         :param pair: Pair that's currently validated
-        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :param ticker: ticker dict as returned from ccxt.fetch_tickers()
         :return: True if the pair can stay, false if it should be removed
         """
         # Check symbol in cache

freqtrade/plugins/pairlistmanager.py

@@ -3,7 +3,7 @@ PairList manager class
 """
 import logging
 from copy import deepcopy
-from typing import Any, Dict, List
+from typing import Dict, List
 
 from cachetools import TTLCache, cached
 
@@ -79,11 +79,8 @@ class PairListManager():
         if self._tickers_needed:
             tickers = self._get_cached_tickers()
 
-        # Adjust whitelist if filters are using tickers
-        pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers)
-
         # Generate the pairlist with first Pairlist Handler in the chain
-        pairlist = self._pairlist_handlers[0].gen_pairlist(self._whitelist, tickers)
+        pairlist = self._pairlist_handlers[0].gen_pairlist(tickers)
 
         # Process all Pairlist Handlers in the chain
         for pairlist_handler in self._pairlist_handlers:
@@ -95,19 +92,6 @@ class PairListManager():
 
         self._whitelist = pairlist
 
-    def _prepare_whitelist(self, pairlist: List[str], tickers: Dict[str, Any]) -> List[str]:
-        """
-        Prepare sanitized pairlist for Pairlist Handlers that use tickers data - remove
-        pairs that do not have ticker available
-        """
-        if self._tickers_needed:
-            # Copy list since we're modifying this list
-            for p in deepcopy(pairlist):
-                if p not in tickers:
-                    pairlist.remove(p)
-
-        return pairlist
-
     def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]:
         """
         Verify and remove items from pairlist - returning a filtered pairlist.

freqtrade/plugins/protections/max_drawdown_protection.py

@@ -61,7 +61,7 @@ class MaxDrawdown(IProtection):
 
         if drawdown > self._max_allowed_drawdown:
             self.log_once(
-                f"Trading stopped due to Max Drawdown {drawdown:.2f} < {self._max_allowed_drawdown}"
+                f"Trading stopped due to Max Drawdown {drawdown:.2f} > {self._max_allowed_drawdown}"
                 f" within {self.lookback_period_str}.", logger.info)
             until = self.calculate_lock_end(trades, self._stop_duration)
 

freqtrade/resolvers/iresolver.py

@@ -61,7 +61,7 @@ class IResolver:
         module = importlib.util.module_from_spec(spec)
         try:
             spec.loader.exec_module(module)  # type: ignore # importlib does not use typehints
-        except (ModuleNotFoundError, SyntaxError, ImportError) as err:
+        except (ModuleNotFoundError, SyntaxError, ImportError, NameError) as err:
             # Catch errors in case a specific module is not installed
             logger.warning(f"Could not import {module_path} due to '{err}'")
             if enum_failed:

freqtrade/rpc/api_server/api_schemas.py

@@ -57,6 +57,7 @@ class Count(BaseModel):
 class PerformanceEntry(BaseModel):
     pair: str
     profit: float
+    profit_abs: float
     count: int
 
 
@@ -151,13 +152,11 @@ class TradeSchema(BaseModel):
     fee_close: Optional[float]
     fee_close_cost: Optional[float]
     fee_close_currency: Optional[str]
-    open_date_hum: str
     open_date: str
     open_timestamp: int
     open_rate: float
     open_rate_requested: Optional[float]
     open_trade_value: float
-    close_date_hum: Optional[str]
     close_date: Optional[str]
     close_timestamp: Optional[int]
     close_rate: Optional[float]
@@ -191,7 +190,6 @@ class OpenTradeSchema(TradeSchema):
     stoploss_current_dist_ratio: Optional[float]
     stoploss_entry_dist: Optional[float]
     stoploss_entry_dist_ratio: Optional[float]
-    base_currency: str
     current_profit: float
     current_profit_abs: float
     current_profit_pct: float
@@ -202,6 +200,7 @@ class OpenTradeSchema(TradeSchema):
 class TradeResponse(BaseModel):
     trades: List[TradeSchema]
     trades_count: int
+    total_trades: int
 
 
 class ForceBuyResponse(BaseModel):
@@ -270,7 +269,7 @@ class DeleteTrade(BaseModel):
 
 class PlotConfig_(BaseModel):
     main_plot: Dict[str, Any]
-    subplots: Optional[Dict[str, Any]]
+    subplots: Dict[str, Any]
 
 
 class PlotConfig(BaseModel):

freqtrade/rpc/api_server/api_v1.py

@@ -17,8 +17,7 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac
                                                   OpenTradeSchema, PairHistory, PerformanceEntry,
                                                   Ping, PlotConfig, Profit, ResultMsg, ShowConfig,
                                                   Stats, StatusMsg, StrategyListResponse,
-                                                  StrategyResponse, TradeResponse, Version,
+                                                  StrategyResponse, Version, WhitelistResponse)
-                                                  WhitelistResponse)
 from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional
 from freqtrade.rpc.rpc import RPCException
 
@@ -83,9 +82,19 @@ def status(rpc: RPC = Depends(get_rpc)):
         return []
 
 
-@router.get('/trades', response_model=TradeResponse, tags=['info', 'trading'])
+# Using the responsemodel here will cause a ~100% increase in response time (from 1s to 2s)
-def trades(limit: int = 0, rpc: RPC = Depends(get_rpc)):
+# on big databases. Correct response model: response_model=TradeResponse,
-    return rpc._rpc_trade_history(limit)
+@router.get('/trades', tags=['info', 'trading'])
+def trades(limit: int = 500, offset: int = 0, rpc: RPC = Depends(get_rpc)):
+    return rpc._rpc_trade_history(limit, offset=offset, order_by_id=True)
+
+
+@router.get('/trade/{tradeid}', response_model=OpenTradeSchema, tags=['info', 'trading'])
+def trade(tradeid: int = 0, rpc: RPC = Depends(get_rpc)):
+    try:
+        return rpc._rpc_trade_status([tradeid])[0]
+    except (RPCException, KeyError):
+        raise HTTPException(status_code=404, detail='Trade not found.')
 
 
 @router.delete('/trades/{tradeid}', response_model=DeleteTrade, tags=['info', 'trading'])

freqtrade/rpc/fiat_convert.py

@@ -3,11 +3,13 @@ Module that define classes to convert Crypto-currency to FIAT
 e.g BTC to USD
 """
 
+import datetime
 import logging
 from typing import Dict
 
 from cachetools.ttl import TTLCache
 from pycoingecko import CoinGeckoAPI
+from requests.exceptions import RequestException
 
 from freqtrade.constants import SUPPORTED_FIAT
 
@@ -25,6 +27,7 @@ class CryptoToFiatConverter:
     _coingekko: CoinGeckoAPI = None
 
     _cryptomap: Dict = {}
+    _backoff: float = 0.0
 
     def __new__(cls):
         """
@@ -47,8 +50,21 @@ class CryptoToFiatConverter:
     def _load_cryptomap(self) -> None:
         try:
             coinlistings = self._coingekko.get_coins_list()
-            # Create mapping table from synbol to coingekko_id
+            # Create mapping table from symbol to coingekko_id
             self._cryptomap = {x['symbol']: x['id'] for x in coinlistings}
+        except RequestException as request_exception:
+            if "429" in str(request_exception):
+                logger.warning(
+                    "Too many requests for Coingecko API, backing off and trying again later.")
+                # Set backoff timestamp to 60 seconds in the future
+                self._backoff = datetime.datetime.now().timestamp() + 60
+                return
+            # If the request is not a 429 error we want to raise the normal error
+            logger.error(
+                "Could not load FIAT Cryptocurrency map for the following problem: {}".format(
+                  request_exception
+                )
+            )
         except (Exception) as exception:
             logger.error(
                 f"Could not load FIAT Cryptocurrency map for the following problem: {exception}")
@@ -127,6 +143,15 @@ class CryptoToFiatConverter:
         if crypto_symbol == fiat_symbol:
             return 1.0
 
+        if self._cryptomap == {}:
+            if self._backoff <= datetime.datetime.now().timestamp():
+                self._load_cryptomap()
+                # return 0.0 if we still dont have data to check, no reason to proceed
+                if self._cryptomap == {}:
+                    return 0.0
+            else:
+                return 0.0
+
         if crypto_symbol not in self._cryptomap:
             # return 0 for unsupported stake currencies (fiat-convert should not break the bot)
             logger.warning("unsupported crypto-symbol %s - returning 0.0", crypto_symbol)

freqtrade/rpc/rpc.py

@@ -24,20 +24,22 @@ from freqtrade.persistence.models import PairLock
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
 from freqtrade.state import State
-from freqtrade.strategy.interface import SellType
+from freqtrade.strategy.interface import SellCheckTuple, SellType
 
 
 logger = logging.getLogger(__name__)
 
 
 class RPCMessageType(Enum):
-    STATUS_NOTIFICATION = 'status'
+    STATUS = 'status'
-    WARNING_NOTIFICATION = 'warning'
+    WARNING = 'warning'
-    STARTUP_NOTIFICATION = 'startup'
+    STARTUP = 'startup'
-    BUY_NOTIFICATION = 'buy'
+    BUY = 'buy'
-    BUY_CANCEL_NOTIFICATION = 'buy_cancel'
+    BUY_FILL = 'buy_fill'
-    SELL_NOTIFICATION = 'sell'
+    BUY_CANCEL = 'buy_cancel'
-    SELL_CANCEL_NOTIFICATION = 'sell_cancel'
+    SELL = 'sell'
+    SELL_FILL = 'sell_fill'
+    SELL_CANCEL = 'sell_cancel'
 
     def __repr__(self):
         return self.value
@@ -167,13 +169,16 @@ class RPC:
                 if trade.open_order_id:
                     order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair)
                 # calculate profit and send message to user
+                if trade.is_open:
                     try:
                         current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
                     except (ExchangeError, PricingError):
                         current_rate = NAN
+                else:
+                    current_rate = trade.close_rate
                 current_profit = trade.calc_profit_ratio(current_rate)
                 current_profit_abs = trade.calc_profit(current_rate)
-
+                current_profit_fiat: Optional[float] = None
                 # Calculate fiat profit
                 if self._fiat_converter:
                     current_profit_fiat = self._fiat_converter.convert_amount(
@@ -215,12 +220,13 @@ class RPC:
             return results
 
     def _rpc_status_table(self, stake_currency: str,
-                          fiat_display_currency: str) -> Tuple[List, List]:
+                          fiat_display_currency: str) -> Tuple[List, List, float]:
         trades = Trade.get_open_trades()
         if not trades:
             raise RPCException('no active trade')
         else:
             trades_list = []
+            fiat_profit_sum = NAN
             for trade in trades:
                 # calculate profit and send message to user
                 try:
@@ -238,6 +244,8 @@ class RPC:
                     )
                     if fiat_profit and not isnan(fiat_profit):
                         profit_str += f" ({fiat_profit:.2f})"
+                        fiat_profit_sum = fiat_profit if isnan(fiat_profit_sum) \
+                            else fiat_profit_sum + fiat_profit
                 trades_list.append([
                     trade.id,
                     trade.pair + ('*' if (trade.open_order_id is not None
@@ -251,7 +259,7 @@ class RPC:
                 profitcol += " (" + fiat_display_currency + ")"
 
             columns = ['ID', 'Pair', 'Since', profitcol]
-            return trades_list, columns
+            return trades_list, columns, fiat_profit_sum
 
     def _rpc_daily_profit(
             self, timescale: int,
@@ -295,11 +303,12 @@ class RPC:
             'data': data
         }
 
-    def _rpc_trade_history(self, limit: int) -> Dict:
+    def _rpc_trade_history(self, limit: int, offset: int = 0, order_by_id: bool = False) -> Dict:
         """ Returns the X last trades """
-        if limit > 0:
+        order_by = Trade.id if order_by_id else Trade.close_date.desc()
+        if limit:
             trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(
-                Trade.close_date.desc()).limit(limit)
+                order_by).limit(limit).offset(offset)
         else:
             trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(
                 Trade.close_date.desc()).all()
@@ -308,7 +317,8 @@ class RPC:
 
         return {
             "trades": output,
-            "trades_count": len(output)
+            "trades_count": len(output),
+            "total_trades": Trade.get_trades([Trade.is_open.is_(False)]).count(),
         }
 
     def _rpc_stats(self) -> Dict[str, Any]:
@@ -442,7 +452,7 @@ class RPC:
         output = []
         total = 0.0
         try:
-            tickers = self._freqtrade.exchange.get_tickers()
+            tickers = self._freqtrade.exchange.get_tickers(cached=True)
         except (ExchangeError):
             raise RPCException('Error getting current tickers.')
 
@@ -547,7 +557,8 @@ class RPC:
             if not fully_canceled:
                 # Get current rate and execute sell
                 current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
-                self._freqtrade.execute_sell(trade, current_rate, SellType.FORCE_SELL)
+                sell_reason = SellCheckTuple(sell_type=SellType.FORCE_SELL)
+                self._freqtrade.execute_sell(trade, current_rate, sell_reason)
         # ---- EOF def _exec_forcesell ----
 
         if self._freqtrade.state != State.RUNNING:
@@ -600,8 +611,7 @@ class RPC:
             raise RPCException(f'position for {pair} already open - id: {trade.id}')
 
         # gen stake amount
-        stakeamount = self._freqtrade.wallets.get_trade_stake_amount(
+        stakeamount = self._freqtrade.wallets.get_trade_stake_amount(pair)
-            pair, self._freqtrade.get_free_open_trades())
 
         # execute buy
         if self._freqtrade.execute_buy(pair, stakeamount, price, forcebuy=True):
@@ -838,5 +848,7 @@ class RPC:
                                               df_analyzed, arrow.Arrow.utcnow().datetime)
 
     def _rpc_plot_config(self) -> Dict[str, Any]:
-
+        if (self._freqtrade.strategy.plot_config and
+                'subplots' not in self._freqtrade.strategy.plot_config):
+            self._freqtrade.strategy.plot_config['subplots'] = {}
         return self._freqtrade.strategy.plot_config

freqtrade/rpc/rpc_manager.py

@@ -67,7 +67,7 @@ class RPCManager:
     def startup_messages(self, config: Dict[str, Any], pairlist, protections) -> None:
         if config['dry_run']:
             self.send_msg({
-                'type': RPCMessageType.WARNING_NOTIFICATION,
+                'type': RPCMessageType.WARNING,
                 'status': 'Dry run is enabled. All trades are simulated.'
             })
         stake_currency = config['stake_currency']
@@ -79,7 +79,7 @@ class RPCManager:
         exchange_name = config['exchange']['name']
         strategy_name = config.get('strategy', '')
         self.send_msg({
-            'type': RPCMessageType.STARTUP_NOTIFICATION,
+            'type': RPCMessageType.STARTUP,
             'status': f'*Exchange:* `{exchange_name}`\n'
                       f'*Stake per trade:* `{stake_amount} {stake_currency}`\n'
                       f'*Minimum ROI:* `{minimal_roi}`\n'
@@ -88,13 +88,13 @@ class RPCManager:
                       f'*Strategy:* `{strategy_name}`'
         })
         self.send_msg({
-            'type': RPCMessageType.STARTUP_NOTIFICATION,
+            'type': RPCMessageType.STARTUP,
             'status': f'Searching for {stake_currency} pairs to buy and sell '
                       f'based on {pairlist.short_desc()}'
         })
         if len(protections.name_list) > 0:
             prots = '\n'.join([p for prot in protections.short_desc() for k, p in prot.items()])
             self.send_msg({
-                'type': RPCMessageType.STARTUP_NOTIFICATION,
+                'type': RPCMessageType.STARTUP,
                 'status': f'Using Protections: \n{prots}'
             })

freqtrade/rpc/telegram.py

@@ -8,19 +8,21 @@ import logging
 from datetime import timedelta
 from html import escape
 from itertools import chain
-from typing import Any, Callable, Dict, List, Union
+from math import isnan
+from typing import Any, Callable, Dict, List, Optional, Union, cast
 
 import arrow
 from tabulate import tabulate
-from telegram import KeyboardButton, ParseMode, ReplyKeyboardMarkup, Update
+from telegram import (InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, ParseMode,
+                      ReplyKeyboardMarkup, Update)
 from telegram.error import NetworkError, TelegramError
-from telegram.ext import CallbackContext, CommandHandler, Updater
+from telegram.ext import CallbackContext, CallbackQueryHandler, CommandHandler, Updater
 from telegram.utils.helpers import escape_markdown
 
 from freqtrade.__init__ import __version__
 from freqtrade.constants import DUST_PER_COIN
 from freqtrade.exceptions import OperationalException
-from freqtrade.misc import round_coin_value
+from freqtrade.misc import chunks, round_coin_value
 from freqtrade.rpc import RPC, RPCException, RPCHandler, RPCMessageType
 
 
@@ -87,7 +89,7 @@ class Telegram(RPCHandler):
         Validates the keyboard configuration from telegram config
         section.
         """
-        self._keyboard: List[List[Union[str, KeyboardButton]]] = [
+        self._keyboard: List[List[Union[str, KeyboardButton, InlineKeyboardButton]]] = [
             ['/daily', '/profit', '/balance'],
             ['/status', '/status table', '/performance'],
             ['/count', '/start', '/stop', '/help']
@@ -169,6 +171,11 @@ class Telegram(RPCHandler):
             [h.command for h in handles]
         )
 
+        self._current_callback_query_handler: Optional[CallbackQueryHandler] = None
+        self._callback_query_handlers = {
+            'forcebuy': CallbackQueryHandler(self._forcebuy_inline)
+        }
+
     def cleanup(self) -> None:
         """
         Stops all running telegram threads.
@@ -176,17 +183,7 @@ class Telegram(RPCHandler):
         """
         self._updater.stop()
 
-    def send_msg(self, msg: Dict[str, Any]) -> None:
+    def _format_buy_msg(self, msg: Dict[str, Any]) -> str:
-        """ Send a message to telegram channel """
-
-        noti = self._config['telegram'].get('notification_settings', {}
-                                            ).get(str(msg['type']), 'on')
-        if noti == 'off':
-            logger.info(f"Notification '{msg['type']}' not sent.")
-            # Notification disabled
-            return
-
-        if msg['type'] == RPCMessageType.BUY_NOTIFICATION:
         if self._rpc._fiat_converter:
             msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount(
                 msg['stake_amount'], msg['stake_currency'], msg['fiat_currency'])
@@ -203,13 +200,9 @@ class Telegram(RPCHandler):
         if msg.get('fiat_currency', None):
             message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}"
         message += ")`"
+        return message
 
-        elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION:
+    def _format_sell_msg(self, msg: Dict[str, Any]) -> str:
-            message = ("\N{WARNING SIGN} *{exchange}:* "
-                       "Cancelling open buy Order for {pair} (#{trade_id}). "
-                       "Reason: {reason}.".format(**msg))
-
-        elif msg['type'] == RPCMessageType.SELL_NOTIFICATION:
         msg['amount'] = round(msg['amount'], 8)
         msg['profit_percent'] = round(msg['profit_ratio'] * 100, 2)
         msg['duration'] = msg['close_date'].replace(
@@ -235,18 +228,45 @@ class Telegram(RPCHandler):
                 msg['profit_amount'], msg['stake_currency'], msg['fiat_currency'])
             message += (' `({gain}: {profit_amount:.8f} {stake_currency}'
                         ' / {profit_fiat:.3f} {fiat_currency})`').format(**msg)
+        return message
 
-        elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION:
+    def send_msg(self, msg: Dict[str, Any]) -> None:
-            message = ("\N{WARNING SIGN} *{exchange}:* Cancelling Open Sell Order "
+        """ Send a message to telegram channel """
-                       "for {pair} (#{trade_id}). Reason: {reason}").format(**msg)
 
-        elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION:
+        noti = self._config['telegram'].get('notification_settings', {}
+                                            ).get(str(msg['type']), 'on')
+        if noti == 'off':
+            logger.info(f"Notification '{msg['type']}' not sent.")
+            # Notification disabled
+            return
+
+        if msg['type'] == RPCMessageType.BUY:
+            message = self._format_buy_msg(msg)
+
+        elif msg['type'] in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL):
+            msg['message_side'] = 'buy' if msg['type'] == RPCMessageType.BUY_CANCEL else 'sell'
+            message = ("\N{WARNING SIGN} *{exchange}:* "
+                       "Cancelling open {message_side} Order for {pair} (#{trade_id}). "
+                       "Reason: {reason}.".format(**msg))
+
+        elif msg['type'] == RPCMessageType.BUY_FILL:
+            message = ("\N{LARGE CIRCLE} *{exchange}:* "
+                       "Buy order for {pair} (#{trade_id}) filled "
+                       "for {open_rate}.".format(**msg))
+        elif msg['type'] == RPCMessageType.SELL_FILL:
+            message = ("\N{LARGE CIRCLE} *{exchange}:* "
+                       "Sell order for {pair} (#{trade_id}) filled "
+                       "for {close_rate}.".format(**msg))
+        elif msg['type'] == RPCMessageType.SELL:
+            message = self._format_sell_msg(msg)
+
+        elif msg['type'] == RPCMessageType.STATUS:
             message = '*Status:* `{status}`'.format(**msg)
 
-        elif msg['type'] == RPCMessageType.WARNING_NOTIFICATION:
+        elif msg['type'] == RPCMessageType.WARNING:
             message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg)
 
-        elif msg['type'] == RPCMessageType.STARTUP_NOTIFICATION:
+        elif msg['type'] == RPCMessageType.STARTUP:
             message = '{status}'.format(**msg)
 
         else:
@@ -294,6 +314,7 @@ class Telegram(RPCHandler):
 
             messages = []
             for r in results:
+                r['open_date_hum'] = arrow.get(r['open_date']).humanize()
                 lines = [
                     "*Trade ID:* `{trade_id}` `(since {open_date_hum})`",
                     "*Current Pair:* {pair}",
@@ -340,19 +361,31 @@ class Telegram(RPCHandler):
         :return: None
         """
         try:
-            statlist, head = self._rpc._rpc_status_table(
+            fiat_currency = self._config.get('fiat_display_currency', '')
-                self._config['stake_currency'], self._config.get('fiat_display_currency', ''))
+            statlist, head, fiat_profit_sum = self._rpc._rpc_status_table(
+                self._config['stake_currency'], fiat_currency)
 
+            show_total = not isnan(fiat_profit_sum) and len(statlist) > 1
             max_trades_per_msg = 50
             """
             Calculate the number of messages of 50 trades per message
             0.99 is used to make sure that there are no extra (empty) messages
             As an example with 50 trades, there will be int(50/50 + 0.99) = 1 message
             """
-            for i in range(0, max(int(len(statlist) / max_trades_per_msg + 0.99), 1)):
+            messages_count = max(int(len(statlist) / max_trades_per_msg + 0.99), 1)
-                message = tabulate(statlist[i * max_trades_per_msg:(i + 1) * max_trades_per_msg],
+            for i in range(0, messages_count):
+                trades = statlist[i * max_trades_per_msg:(i + 1) * max_trades_per_msg]
+                if show_total and i == messages_count - 1:
+                    # append total line
+                    trades.append(["Total", "", "", f"{fiat_profit_sum:.2f} {fiat_currency}"])
+
+                message = tabulate(trades,
                                    headers=head,
                                    tablefmt='simple')
+                if show_total and i == messages_count - 1:
+                    # insert separators line between Total
+                    lines = message.split("\n")
+                    message = "\n".join(lines[:-1] + [lines[1]] + [lines[-1]])
                 self._send_msg(f"<pre>{message}</pre>", parse_mode=ParseMode.HTML)
         except RPCException as e:
             self._send_msg(str(e))
@@ -610,6 +643,25 @@ class Telegram(RPCHandler):
         except RPCException as e:
             self._send_msg(str(e))
 
+    def _forcebuy_action(self, pair, price=None):
+        try:
+            self._rpc._rpc_forcebuy(pair, price)
+        except RPCException as e:
+            self._send_msg(str(e))
+
+    def _forcebuy_inline(self, update: Update, _: CallbackContext) -> None:
+        if update.callback_query:
+            query = update.callback_query
+            pair = query.data
+            query.answer()
+            query.edit_message_text(text=f"Force Buying: {pair}")
+            self._forcebuy_action(pair)
+
+    @staticmethod
+    def _layout_inline_keyboard(buttons: List[InlineKeyboardButton],
+                                cols=3) -> List[List[InlineKeyboardButton]]:
+        return [buttons[i:i + cols] for i in range(0, len(buttons), cols)]
+
     @authorized_only
     def _forcebuy(self, update: Update, context: CallbackContext) -> None:
         """
@@ -622,10 +674,13 @@ class Telegram(RPCHandler):
         if context.args:
             pair = context.args[0]
             price = float(context.args[1]) if len(context.args) > 1 else None
-            try:
+            self._forcebuy_action(pair, price)
-                self._rpc._rpc_forcebuy(pair, price)
+        else:
-            except RPCException as e:
+            whitelist = self._rpc._rpc_whitelist()['whitelist']
-                self._send_msg(str(e))
+            pairs = [InlineKeyboardButton(pair, callback_data=pair) for pair in whitelist]
+            self._send_inline_msg("Which pair?",
+                                  keyboard=self._layout_inline_keyboard(pairs),
+                                  callback_query_handler='forcebuy')
 
     @authorized_only
     def _trades(self, update: Update, context: CallbackContext) -> None:
@@ -697,11 +752,14 @@ class Telegram(RPCHandler):
             trades = self._rpc._rpc_performance()
             output = "<b>Performance:</b>\n"
             for i, trade in enumerate(trades):
-                stat_line = (f"{i+1}.\t <code>{trade['pair']}\t{trade['profit']:.2f}% "
+                stat_line = (
+                    f"{i+1}.\t <code>{trade['pair']}\t"
+                    f"{round_coin_value(trade['profit_abs'], self._config['stake_currency'])} "
+                    f"({trade['profit']:.2f}%) "
                     f"({trade['count']})</code>\n")
 
                 if len(output + stat_line) >= MAX_TELEGRAM_MESSAGE_LENGTH:
-                    self._send_msg(output)
+                    self._send_msg(output, parse_mode=ParseMode.HTML)
                     output = stat_line
                 else:
                     output += stat_line
@@ -736,12 +794,16 @@ class Telegram(RPCHandler):
         Handler for /locks.
         Returns the currently active locks
         """
-        locks = self._rpc._rpc_locks()
+        rpc_locks = self._rpc._rpc_locks()
+        if not rpc_locks['locks']:
+            self._send_msg('No active locks.', parse_mode=ParseMode.HTML)
+
+        for locks in chunks(rpc_locks['locks'], 25):
             message = tabulate([[
                 lock['id'],
                 lock['pair'],
                 lock['lock_end_time'],
-            lock['reason']] for lock in locks['locks']],
+                lock['reason']] for lock in locks],
                 headers=['ID', 'Pair', 'Until', 'Reason'],
                 tablefmt='simple')
             message = f"<pre>{escape(message)}</pre>"
@@ -846,9 +908,17 @@ class Telegram(RPCHandler):
         """
         try:
             edge_pairs = self._rpc._rpc_edge()
-            edge_pairs_tab = tabulate(edge_pairs, headers='keys', tablefmt='simple')
+            if not edge_pairs:
-            message = f'<b>Edge only validated following pairs:</b>\n<pre>{edge_pairs_tab}</pre>'
+                message = '<b>Edge only validated following pairs:</b>'
                 self._send_msg(message, parse_mode=ParseMode.HTML)
+
+            for chunk in chunks(edge_pairs, 25):
+                edge_pairs_tab = tabulate(chunk, headers='keys', tablefmt='simple')
+                message = (f'<b>Edge only validated following pairs:</b>\n'
+                           f'<pre>{edge_pairs_tab}</pre>')
+
+                self._send_msg(message, parse_mode=ParseMode.HTML)
+
         except RPCException as e:
             self._send_msg(str(e))
 
@@ -945,8 +1015,9 @@ class Telegram(RPCHandler):
             f"*Current state:* `{val['state']}`"
         )
 
-    def _send_msg(self, msg: str, parse_mode: str = ParseMode.MARKDOWN,
+    def _send_inline_msg(self, msg: str, callback_query_handler,
-                  disable_notification: bool = False) -> None:
+                         parse_mode: str = ParseMode.MARKDOWN, disable_notification: bool = False,
+                         keyboard: List[List[InlineKeyboardButton]] = None, ) -> None:
         """
         Send given markdown message
         :param msg: message
@@ -954,7 +1025,29 @@ class Telegram(RPCHandler):
         :param parse_mode: telegram parse mode
         :return: None
         """
-        reply_markup = ReplyKeyboardMarkup(self._keyboard, resize_keyboard=True)
+        if self._current_callback_query_handler:
+            self._updater.dispatcher.remove_handler(self._current_callback_query_handler)
+        self._current_callback_query_handler = self._callback_query_handlers[callback_query_handler]
+        self._updater.dispatcher.add_handler(self._current_callback_query_handler)
+
+        self._send_msg(msg, parse_mode, disable_notification,
+                       cast(List[List[Union[str, KeyboardButton, InlineKeyboardButton]]], keyboard),
+                       reply_markup=InlineKeyboardMarkup)
+
+    def _send_msg(self, msg: str, parse_mode: str = ParseMode.MARKDOWN,
+                  disable_notification: bool = False,
+                  keyboard: List[List[Union[str, KeyboardButton, InlineKeyboardButton]]] = None,
+                  reply_markup=ReplyKeyboardMarkup) -> None:
+        """
+        Send given markdown message
+        :param msg: message
+        :param bot: alternative bot
+        :param parse_mode: telegram parse mode
+        :return: None
+        """
+        if keyboard is None:
+            keyboard = self._keyboard
+        reply_markup = reply_markup(keyboard, resize_keyboard=True)
         try:
             try:
                 self._updater.bot.send_message(

freqtrade/rpc/webhook.py

@@ -45,17 +45,21 @@ class Webhook(RPCHandler):
         """ Send a message to telegram channel """
         try:
 
-            if msg['type'] == RPCMessageType.BUY_NOTIFICATION:
+            if msg['type'] == RPCMessageType.BUY:
                 valuedict = self._config['webhook'].get('webhookbuy', None)
-            elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION:
+            elif msg['type'] == RPCMessageType.BUY_CANCEL:
                 valuedict = self._config['webhook'].get('webhookbuycancel', None)
-            elif msg['type'] == RPCMessageType.SELL_NOTIFICATION:
+            elif msg['type'] == RPCMessageType.BUY_FILL:
+                valuedict = self._config['webhook'].get('webhookbuyfill', None)
+            elif msg['type'] == RPCMessageType.SELL:
                 valuedict = self._config['webhook'].get('webhooksell', None)
-            elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION:
+            elif msg['type'] == RPCMessageType.SELL_FILL:
+                valuedict = self._config['webhook'].get('webhooksellfill', None)
+            elif msg['type'] == RPCMessageType.SELL_CANCEL:
                 valuedict = self._config['webhook'].get('webhooksellcancel', None)
-            elif msg['type'] in (RPCMessageType.STATUS_NOTIFICATION,
+            elif msg['type'] in (RPCMessageType.STATUS,
-                                 RPCMessageType.STARTUP_NOTIFICATION,
+                                 RPCMessageType.STARTUP,
-                                 RPCMessageType.WARNING_NOTIFICATION):
+                                 RPCMessageType.WARNING):
                 valuedict = self._config['webhook'].get('webhookstatus', None)
             else:
                 raise NotImplementedError('Unknown message type: {}'.format(msg['type']))

freqtrade/strategy/hyper.py

@@ -5,13 +5,17 @@ This module defines a base class for auto-hyperoptable strategies.
 import logging
 from abc import ABC, abstractmethod
 from contextlib import suppress
-from typing import Any, Iterator, Optional, Sequence, Tuple, Union
+from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union
+
+from freqtrade.optimize.hyperopt_tools import HyperoptTools
 
 
 with suppress(ImportError):
     from skopt.space import Integer, Real, Categorical
+    from freqtrade.optimize.space import SKDecimal
 
 from freqtrade.exceptions import OperationalException
+from freqtrade.state import RunMode
 
 
 logger = logging.getLogger(__name__)
@@ -24,9 +28,10 @@ class BaseParameter(ABC):
     category: Optional[str]
     default: Any
     value: Any
-    opt_range: Sequence[Any]
+    in_space: bool = False
+    name: str
 
-    def __init__(self, *, opt_range: Sequence[Any], default: Any, space: Optional[str] = None,
+    def __init__(self, *, default: Any, space: Optional[str] = None,
                  optimize: bool = True, load: bool = True, **kwargs):
         """
         Initialize hyperopt-optimizable parameter.
@@ -43,7 +48,6 @@ class BaseParameter(ABC):
         self.category = space
         self._space_params = kwargs
         self.value = default
-        self.opt_range = opt_range
         self.optimize = optimize
         self.load = load
 
@@ -51,24 +55,51 @@ class BaseParameter(ABC):
         return f'{self.__class__.__name__}({self.value})'
 
     @abstractmethod
-    def get_space(self, name: str) -> Union['Integer', 'Real', 'Categorical']:
+    def get_space(self, name: str) -> Union['Integer', 'Real', 'SKDecimal', 'Categorical']:
         """
         Get-space - will be used by Hyperopt to get the hyperopt Space
         """
 
-    def _set_value(self, value: Any):
+
+class NumericParameter(BaseParameter):
+    """ Internal parameter used for Numeric purposes """
+    float_or_int = Union[int, float]
+    default: float_or_int
+    value: float_or_int
+
+    def __init__(self, low: Union[float_or_int, Sequence[float_or_int]],
+                 high: Optional[float_or_int] = None, *, default: float_or_int,
+                 space: Optional[str] = None, optimize: bool = True, load: bool = True, **kwargs):
         """
-        Update current value. Used by hyperopt functions for the purpose where optimization and
+        Initialize hyperopt-optimizable numeric parameter.
-         value spaces differ.
+        Cannot be instantiated, but provides the validation for other numeric parameters
-        :param value: A numerical value.
+        :param low: Lower end (inclusive) of optimization space or [low, high].
+        :param high: Upper end (inclusive) of optimization space.
+                     Must be none of entire range is passed first parameter.
+        :param default: A default value.
+        :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if
+                      parameter fieldname is prefixed with 'buy_' or 'sell_'.
+        :param optimize: Include parameter in hyperopt optimizations.
+        :param load: Load parameter value from {space}_params.
+        :param kwargs: Extra parameters to skopt.space.*.
         """
-        self.value = value
+        if high is not None and isinstance(low, Sequence):
+            raise OperationalException(f'{self.__class__.__name__} space invalid.')
+        if high is None or isinstance(low, Sequence):
+            if not isinstance(low, Sequence) or len(low) != 2:
+                raise OperationalException(f'{self.__class__.__name__} space must be [low, high]')
+            self.low, self.high = low
+        else:
+            self.low = low
+            self.high = high
+
+        super().__init__(default=default, space=space, optimize=optimize,
+                         load=load, **kwargs)
 
 
-class IntParameter(BaseParameter):
+class IntParameter(NumericParameter):
     default: int
     value: int
-    opt_range: Sequence[int]
 
     def __init__(self, low: Union[int, Sequence[int]], high: Optional[int] = None, *, default: int,
                  space: Optional[str] = None, optimize: bool = True, load: bool = True, **kwargs):
@@ -84,15 +115,8 @@ class IntParameter(BaseParameter):
         :param load: Load parameter value from {space}_params.
         :param kwargs: Extra parameters to skopt.space.Integer.
         """
-        if high is not None and isinstance(low, Sequence):
+
-            raise OperationalException('IntParameter space invalid.')
+        super().__init__(low=low, high=high, default=default, space=space, optimize=optimize,
-        if high is None or isinstance(low, Sequence):
-            if not isinstance(low, Sequence) or len(low) != 2:
-                raise OperationalException('IntParameter space must be [low, high]')
-            opt_range = low
-        else:
-            opt_range = [low, high]
-        super().__init__(opt_range=opt_range, default=default, space=space, optimize=optimize,
                          load=load, **kwargs)
 
     def get_space(self, name: str) -> 'Integer':
@@ -100,13 +124,26 @@ class IntParameter(BaseParameter):
         Create skopt optimization space.
         :param name: A name of parameter field.
         """
-        return Integer(*self.opt_range, name=name, **self._space_params)
+        return Integer(low=self.low, high=self.high, name=name, **self._space_params)
+
+    @property
+    def range(self):
+        """
+        Get each value in this space as list.
+        Returns a List from low to high (inclusive) in Hyperopt mode.
+        Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid
+        calculating 100ds of indicators.
+        """
+        if self.in_space and self.optimize:
+            # Scikit-optimize ranges are "inclusive", while python's "range" is exclusive
+            return range(self.low, self.high + 1)
+        else:
+            return range(self.value, self.value + 1)
 
 
-class RealParameter(BaseParameter):
+class RealParameter(NumericParameter):
     default: float
     value: float
-    opt_range: Sequence[float]
 
     def __init__(self, low: Union[float, Sequence[float]], high: Optional[float] = None, *,
                  default: float, space: Optional[str] = None, optimize: bool = True,
@@ -123,15 +160,7 @@ class RealParameter(BaseParameter):
         :param load: Load parameter value from {space}_params.
         :param kwargs: Extra parameters to skopt.space.Real.
         """
-        if high is not None and isinstance(low, Sequence):
+        super().__init__(low=low, high=high, default=default, space=space, optimize=optimize,
-            raise OperationalException(f'{self.__class__.__name__} space invalid.')
-        if high is None or isinstance(low, Sequence):
-            if not isinstance(low, Sequence) or len(low) != 2:
-                raise OperationalException(f'{self.__class__.__name__} space must be [low, high]')
-            opt_range = low
-        else:
-            opt_range = [low, high]
-        super().__init__(opt_range=opt_range, default=default, space=space, optimize=optimize,
                          load=load, **kwargs)
 
     def get_space(self, name: str) -> 'Real':
@@ -139,13 +168,12 @@ class RealParameter(BaseParameter):
         Create skopt optimization space.
         :param name: A name of parameter field.
         """
-        return Real(*self.opt_range, name=name, **self._space_params)
+        return Real(low=self.low, high=self.high, name=name, **self._space_params)
 
 
-class DecimalParameter(RealParameter):
+class DecimalParameter(NumericParameter):
     default: float
     value: float
-    opt_range: Sequence[float]
 
     def __init__(self, low: Union[float, Sequence[float]], high: Optional[float] = None, *,
                  default: float, decimals: int = 3, space: Optional[str] = None,
@@ -161,29 +189,21 @@ class DecimalParameter(RealParameter):
                       parameter fieldname is prefixed with 'buy_' or 'sell_'.
         :param optimize: Include parameter in hyperopt optimizations.
         :param load: Load parameter value from {space}_params.
-        :param kwargs: Extra parameters to skopt.space.Real.
+        :param kwargs: Extra parameters to skopt.space.Integer.
         """
         self._decimals = decimals
         default = round(default, self._decimals)
+
         super().__init__(low=low, high=high, default=default, space=space, optimize=optimize,
                          load=load, **kwargs)
 
-    def get_space(self, name: str) -> 'Integer':
+    def get_space(self, name: str) -> 'SKDecimal':
         """
         Create skopt optimization space.
         :param name: A name of parameter field.
         """
-        low = int(self.opt_range[0] * pow(10, self._decimals))
+        return SKDecimal(low=self.low, high=self.high, decimals=self._decimals, name=name,
-        high = int(self.opt_range[1] * pow(10, self._decimals))
+                         **self._space_params)
-        return Integer(low, high, name=name, **self._space_params)
-
-    def _set_value(self, value: int):
-        """
-        Update current value. Used by hyperopt functions for the purpose where optimization and
-         value spaces differ.
-        :param value: An integer value.
-        """
-        self.value = round(value * pow(0.1, self._decimals), self._decimals)
 
 
 class CategoricalParameter(BaseParameter):
@@ -208,7 +228,8 @@ class CategoricalParameter(BaseParameter):
         if len(categories) < 2:
             raise OperationalException(
                 'CategoricalParameter space must be [a, b, ...] (at least two parameters)')
-        super().__init__(opt_range=categories, default=default, space=space, optimize=optimize,
+        self.opt_range = categories
+        super().__init__(default=default, space=space, optimize=optimize,
                          load=load, **kwargs)
 
     def get_space(self, name: str) -> 'Categorical':
@@ -225,12 +246,15 @@ class HyperStrategyMixin(object):
      strategy logic.
     """
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, config: Dict[str, Any], *args, **kwargs):
         """
         Initialize hyperoptable strategy mixin.
         """
-        self._load_params(getattr(self, 'buy_params', None))
+        self.config = config
-        self._load_params(getattr(self, 'sell_params', None))
+        self.ft_buy_params: List[BaseParameter] = []
+        self.ft_sell_params: List[BaseParameter] = []
+
+        self._load_hyper_params(config.get('runmode') == RunMode.HYPEROPT)
 
     def enumerate_parameters(self, category: str = None) -> Iterator[Tuple[str, BaseParameter]]:
         """
@@ -240,30 +264,72 @@ class HyperStrategyMixin(object):
         """
         if category not in ('buy', 'sell', None):
             raise OperationalException('Category must be one of: "buy", "sell", None.')
+
+        if category is None:
+            params = self.ft_buy_params + self.ft_sell_params
+        else:
+            params = getattr(self, f"ft_{category}_params")
+
+        for par in params:
+            yield par.name, par
+
+    def _detect_parameters(self, category: str) -> Iterator[Tuple[str, BaseParameter]]:
+        """ Detect all parameters for 'category' """
         for attr_name in dir(self):
             if not attr_name.startswith('__'):  # Ignore internals, not strictly necessary.
                 attr = getattr(self, attr_name)
                 if issubclass(attr.__class__, BaseParameter):
-                    if (category and attr_name.startswith(category + '_')
+                    if (attr_name.startswith(category + '_')
                             and attr.category is not None and attr.category != category):
                         raise OperationalException(
                             f'Inconclusive parameter name {attr_name}, category: {attr.category}.')
-                    if (category is None or category == attr.category or
+                    if (category == attr.category or
                             (attr_name.startswith(category + '_') and attr.category is None)):
                         yield attr_name, attr
 
-    def _load_params(self, params: dict) -> None:
+    def _load_hyper_params(self, hyperopt: bool = False) -> None:
+        """
+        Load Hyperoptable parameters
+        """
+        self._load_params(getattr(self, 'buy_params', None), 'buy', hyperopt)
+        self._load_params(getattr(self, 'sell_params', None), 'sell', hyperopt)
+
+    def _load_params(self, params: dict, space: str, hyperopt: bool = False) -> None:
         """
         Set optimizeable parameter values.
         :param params: Dictionary with new parameter values.
         """
         if not params:
-            return
+            logger.info(f"No params for {space} found, using default values.")
-        for attr_name, attr in self.enumerate_parameters():
+        param_container: List[BaseParameter] = getattr(self, f"ft_{space}_params")
-            if attr_name in params:
+
+        for attr_name, attr in self._detect_parameters(space):
+            attr.name = attr_name
+            attr.in_space = hyperopt and HyperoptTools.has_space(self.config, space)
+            if not attr.category:
+                attr.category = space
+
+            param_container.append(attr)
+
+            if params and attr_name in params:
                 if attr.load:
                     attr.value = params[attr_name]
                     logger.info(f'Strategy Parameter: {attr_name} = {attr.value}')
                 else:
                     logger.warning(f'Parameter "{attr_name}" exists, but is disabled. '
                                    f'Default value "{attr.value}" used.')
+            else:
+                logger.info(f'Strategy Parameter(default): {attr_name} = {attr.value}')
+
+    def get_params_dict(self):
+        """
+        Returns list of Parameters that are not part of the current optimize job
+        """
+        params = {
+            'buy': {},
+            'sell': {}
+        }
+        for name, p in self.enumerate_parameters():
+            if not p.optimize or not p.in_space:
+                params[p.category][name] = p.value
+        return params

freqtrade/strategy/interface.py

@@ -7,7 +7,7 @@ import warnings
 from abc import ABC, abstractmethod
 from datetime import datetime, timedelta, timezone
 from enum import Enum
-from typing import Dict, List, NamedTuple, Optional, Tuple
+from typing import Dict, List, Optional, Tuple, Union
 
 import arrow
 from pandas import DataFrame
@@ -24,6 +24,7 @@ from freqtrade.wallets import Wallets
 
 
 logger = logging.getLogger(__name__)
+CUSTOM_SELL_MAX_LENGTH = 64
 
 
 class SignalType(Enum):
@@ -45,6 +46,7 @@ class SellType(Enum):
     SELL_SIGNAL = "sell_signal"
     FORCE_SELL = "force_sell"
     EMERGENCY_SELL = "emergency_sell"
+    CUSTOM_SELL = "custom_sell"
     NONE = ""
 
     def __str__(self):
@@ -52,12 +54,20 @@ class SellType(Enum):
         return self.value
 
 
-class SellCheckTuple(NamedTuple):
+class SellCheckTuple(object):
     """
     NamedTuple for Sell type + reason
     """
-    sell_flag: bool
     sell_type: SellType
+    sell_reason: str = ''
+
+    def __init__(self, sell_type: SellType, sell_reason: str = ''):
+        self.sell_type = sell_type
+        self.sell_reason = sell_reason or sell_type.value
+
+    @property
+    def sell_flag(self):
+        return self.sell_type != SellType.NONE
 
 
 class IStrategy(ABC, HyperStrategyMixin):
@@ -151,6 +161,7 @@ class IStrategy(ABC, HyperStrategyMixin):
         :param metadata: Additional information, like the currently traded pair
         :return: a Dataframe with all mandatory indicators for the strategies
         """
+        return dataframe
 
     @abstractmethod
     def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
@@ -160,6 +171,7 @@ class IStrategy(ABC, HyperStrategyMixin):
         :param metadata: Additional information, like the currently traded pair
         :return: DataFrame with buy column
         """
+        return dataframe
 
     @abstractmethod
     def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
@@ -169,6 +181,7 @@ class IStrategy(ABC, HyperStrategyMixin):
         :param metadata: Additional information, like the currently traded pair
         :return: DataFrame with sell column
         """
+        return dataframe
 
     def check_buy_timeout(self, pair: str, trade: Trade, order: dict, **kwargs) -> bool:
         """
@@ -216,7 +229,7 @@ class IStrategy(ABC, HyperStrategyMixin):
         pass
 
     def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float,
-                            time_in_force: str, **kwargs) -> bool:
+                            time_in_force: str, current_time: datetime, **kwargs) -> bool:
         """
         Called right before placing a buy order.
         Timing for this function is critical, so avoid doing heavy computations or
@@ -231,6 +244,7 @@ class IStrategy(ABC, HyperStrategyMixin):
         :param amount: Amount in target (quote) currency that's going to be traded.
         :param rate: Rate that's going to be used when using limit orders
         :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled).
+        :param current_time: datetime object, containing the current datetime
         :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
         :return bool: When True is returned, then the buy-order is placed on the exchange.
             False aborts the process
@@ -238,7 +252,8 @@ class IStrategy(ABC, HyperStrategyMixin):
         return True
 
     def confirm_trade_exit(self, pair: str, trade: Trade, order_type: str, amount: float,
-                           rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool:
+                           rate: float, time_in_force: str, sell_reason: str,
+                           current_time: datetime, **kwargs) -> bool:
         """
         Called right before placing a regular sell order.
         Timing for this function is critical, so avoid doing heavy computations or
@@ -257,6 +272,7 @@ class IStrategy(ABC, HyperStrategyMixin):
         :param sell_reason: Sell reason.
             Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss',
                            'sell_signal', 'force_sell', 'emergency_sell']
+        :param current_time: datetime object, containing the current datetime
         :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
         :return bool: When True is returned, then the sell-order is placed on the exchange.
             False aborts the process
@@ -285,6 +301,30 @@ class IStrategy(ABC, HyperStrategyMixin):
         """
         return self.stoploss
 
+    def custom_sell(self, pair: str, trade: Trade, current_time: datetime, current_rate: float,
+                    current_profit: float, **kwargs) -> Optional[Union[str, bool]]:
+        """
+        Custom sell signal logic indicating that specified position should be sold. Returning a
+        string or True from this method is equal to setting sell signal on a candle at specified
+        time. This method is not called when sell signal is set.
+
+        This method should be overridden to create sell signals that depend on trade parameters. For
+        example you could implement a stoploss relative to candle when trade was opened, or a custom
+        1:2 risk-reward ROI.
+
+        Custom sell reason max length is 64. Exceeding this limit will raise OperationalException.
+
+        :param pair: Pair that's currently analyzed
+        :param trade: trade object.
+        :param current_time: datetime object, containing the current datetime
+        :param current_rate: Rate, calculated based on pricing settings in ask_strategy.
+        :param current_profit: Current profit (as ratio), calculated based on current_rate.
+        :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+        :return: To execute sell, return a string with custom sell reason or True. Otherwise return
+        None or False.
+        """
+        return None
+
     def informative_pairs(self) -> ListPairsWithTimeframes:
         """
         Define additional, informative pair/interval combinations to be cached from the exchange.
@@ -531,12 +571,33 @@ class IStrategy(ABC, HyperStrategyMixin):
                        and self.min_roi_reached(trade=trade, current_profit=current_profit,
                                                 current_time=date))
 
+        sell_signal = SellType.NONE
+        custom_reason = ''
+        # use provided rate in backtesting, not high/low.
+        current_rate = rate
+        current_profit = trade.calc_profit_ratio(current_rate)
+
         if (ask_strategy.get('sell_profit_only', False)
                 and current_profit <= ask_strategy.get('sell_profit_offset', 0)):
             # sell_profit_only and profit doesn't reach the offset - ignore sell signal
-            sell_signal = False
+            pass
+        elif ask_strategy.get('use_sell_signal', True) and not buy:
+            if sell:
+                sell_signal = SellType.SELL_SIGNAL
             else:
-            sell_signal = sell and not buy and ask_strategy.get('use_sell_signal', True)
+                custom_reason = strategy_safe_wrapper(self.custom_sell, default_retval=False)(
+                    pair=trade.pair, trade=trade, current_time=date, current_rate=current_rate,
+                    current_profit=current_profit)
+                if custom_reason:
+                    sell_signal = SellType.CUSTOM_SELL
+                    if isinstance(custom_reason, str):
+                        if len(custom_reason) > CUSTOM_SELL_MAX_LENGTH:
+                            logger.warning(f'Custom sell reason returned from custom_sell is too '
+                                           f'long and was trimmed to {CUSTOM_SELL_MAX_LENGTH} '
+                                           f'characters.')
+                            custom_reason = custom_reason[:CUSTOM_SELL_MAX_LENGTH]
+                    else:
+                        custom_reason = None
             # TODO: return here if sell-signal should be favored over ROI
 
         # Start evaluations
@@ -545,24 +606,23 @@ class IStrategy(ABC, HyperStrategyMixin):
         # Sell-signal
         # Stoploss
         if roi_reached and stoplossflag.sell_type != SellType.STOP_LOSS:
-            logger.debug(f"{trade.pair} - Required profit reached. sell_flag=True, "
+            logger.debug(f"{trade.pair} - Required profit reached. sell_type=SellType.ROI")
-                         f"sell_type=SellType.ROI")
+            return SellCheckTuple(sell_type=SellType.ROI)
-            return SellCheckTuple(sell_flag=True, sell_type=SellType.ROI)
 
-        if sell_signal:
+        if sell_signal != SellType.NONE:
-            logger.debug(f"{trade.pair} - Sell signal received. sell_flag=True, "
+            logger.debug(f"{trade.pair} - Sell signal received. "
-                         f"sell_type=SellType.SELL_SIGNAL")
+                         f"sell_type=SellType.{sell_signal.name}" +
-            return SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL)
+                         (f", custom_reason={custom_reason}" if custom_reason else ""))
+            return SellCheckTuple(sell_type=sell_signal, sell_reason=custom_reason)
 
         if stoplossflag.sell_flag:
 
-            logger.debug(f"{trade.pair} - Stoploss hit. sell_flag=True, "
+            logger.debug(f"{trade.pair} - Stoploss hit. sell_type={stoplossflag.sell_type}")
-                         f"sell_type={stoplossflag.sell_type}")
             return stoplossflag
 
         # This one is noisy, commented out...
-        # logger.debug(f"{trade.pair} - No sell signal. sell_flag=False")
+        # logger.debug(f"{trade.pair} - No sell signal.")
-        return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE)
+        return SellCheckTuple(sell_type=SellType.NONE)
 
     def stop_loss_reached(self, current_rate: float, trade: Trade,
                           current_time: datetime, current_profit: float,
@@ -626,9 +686,9 @@ class IStrategy(ABC, HyperStrategyMixin):
                 logger.debug(f"{trade.pair} - Trailing stop saved "
                              f"{trade.stop_loss - trade.initial_stop_loss:.6f}")
 
-            return SellCheckTuple(sell_flag=True, sell_type=sell_type)
+            return SellCheckTuple(sell_type=sell_type)
 
-        return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE)
+        return SellCheckTuple(sell_type=SellType.NONE)
 
     def min_roi_reached_entry(self, trade_dur: int) -> Tuple[Optional[int], Optional[float]]:
         """

freqtrade/templates/base_config.json.j2

@@ -9,7 +9,8 @@
     "cancel_open_orders_on_exit": false,
     "unfilledtimeout": {
         "buy": 10,
-        "sell": 30
+        "sell": 30,
+        "unit": "minutes"
     },
     "bid_strategy": {
         "price_side": "bid",

freqtrade/templates/sample_hyperopt_advanced.py

@@ -7,7 +7,7 @@ from typing import Any, Callable, Dict, List
 import numpy as np  # noqa
 import pandas as pd  # noqa
 from pandas import DataFrame
-from skopt.space import Categorical, Dimension, Integer, Real  # noqa
+from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal, Real  # noqa
 
 from freqtrade.optimize.hyperopt_interface import IHyperOpt
 
@@ -223,9 +223,9 @@ class AdvancedSampleHyperOpt(IHyperOpt):
             Integer(10, 120, name='roi_t1'),
             Integer(10, 60, name='roi_t2'),
             Integer(10, 40, name='roi_t3'),
-            Real(0.01, 0.04, name='roi_p1'),
+            SKDecimal(0.01, 0.04, decimals=3, name='roi_p1'),
-            Real(0.01, 0.07, name='roi_p2'),
+            SKDecimal(0.01, 0.07, decimals=3, name='roi_p2'),
-            Real(0.01, 0.20, name='roi_p3'),
+            SKDecimal(0.01, 0.20, decimals=3, name='roi_p3'),
         ]
 
     @staticmethod
@@ -237,7 +237,7 @@ class AdvancedSampleHyperOpt(IHyperOpt):
         'stoploss' optimization hyperspace.
         """
         return [
-            Real(-0.35, -0.02, name='stoploss'),
+            SKDecimal(-0.35, -0.02, decimals=3, name='stoploss'),
         ]
 
     @staticmethod
@@ -256,14 +256,14 @@ class AdvancedSampleHyperOpt(IHyperOpt):
             # other 'trailing' hyperspace parameters.
             Categorical([True], name='trailing_stop'),
 
-            Real(0.01, 0.35, name='trailing_stop_positive'),
+            SKDecimal(0.01, 0.35, decimals=3, name='trailing_stop_positive'),
 
             # 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive',
             # so this intermediate parameter is used as the value of the difference between
             # them. The value of the 'trailing_stop_positive_offset' is constructed in the
             # generate_trailing_params() method.
             # This is similar to the hyperspace dimensions used for constructing the ROI tables.
-            Real(0.001, 0.1, name='trailing_stop_positive_offset_p1'),
+            SKDecimal(0.001, 0.1, decimals=3, name='trailing_stop_positive_offset_p1'),
 
             Categorical([True, False], name='trailing_only_offset_is_reached'),
         ]

freqtrade/templates/sample_strategy.py

@@ -361,7 +361,7 @@ class SampleStrategy(IStrategy):
         Based on TA indicators, populates the sell signal for the given dataframe
         :param dataframe: DataFrame populated with indicators
         :param metadata: Additional information, like the currently traded pair
-        :return: DataFrame with buy column
+        :return: DataFrame with sell column
         """
         dataframe.loc[
             (

freqtrade/templates/strategy_analysis_example.ipynb

@@ -282,6 +282,28 @@
     "graph.show(renderer=\"browser\")\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Plot average profit per trade as distribution graph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import plotly.figure_factory as ff\n",
+    "\n",
+    "hist_data = [trades.profit_ratio]\n",
+    "group_labels = ['profit_ratio']  # name of the dataset\n",
+    "\n",
+    "fig = ff.create_distplot(hist_data, group_labels,bin_size=0.01)\n",
+    "fig.show()\n"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

freqtrade/templates/subtemplates/strategy_methods_advanced.j2

@@ -14,8 +14,9 @@ def bot_loop_start(self, **kwargs) -> None:
 
 use_custom_stoploss = True
 
-def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float,
+def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime',
-                    current_profit: float, **kwargs) -> float:
+                    current_rate: float, current_profit: float, dataframe: DataFrame,
+                    **kwargs) -> float:
     """
     Custom stoploss logic, returning the new distance relative to current_rate (as ratio).
     e.g. returning -0.05 would create a stoploss 5% below current_rate.
@@ -31,13 +32,14 @@ def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', c
     :param current_time: datetime object, containing the current datetime
     :param current_rate: Rate, calculated based on pricing settings in ask_strategy.
     :param current_profit: Current profit (as ratio), calculated based on current_rate.
+    :param dataframe: Analyzed dataframe for this pair. Can contain future data in backtesting.
     :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
     :return float: New stoploss value, relative to the currentrate
     """
     return self.stoploss
 
 def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float,
-                        time_in_force: str, **kwargs) -> bool:
+                        time_in_force: str, current_time: 'datetime', **kwargs) -> bool:
     """
     Called right before placing a buy order.
     Timing for this function is critical, so avoid doing heavy computations or
@@ -52,6 +54,7 @@ def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: f
     :param amount: Amount in target (quote) currency that's going to be traded.
     :param rate: Rate that's going to be used when using limit orders
     :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled).
+    :param current_time: datetime object, containing the current datetime
     :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
     :return bool: When True is returned, then the buy-order is placed on the exchange.
         False aborts the process
@@ -59,7 +62,8 @@ def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: f
     return True
 
 def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: float,
-                       rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool:
+                       rate: float, time_in_force: str, sell_reason: str,
+                       current_time: 'datetime', **kwargs) -> bool:
     """
     Called right before placing a regular sell order.
     Timing for this function is critical, so avoid doing heavy computations or
@@ -78,6 +82,7 @@ def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount:
     :param sell_reason: Sell reason.
         Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss',
                         'sell_signal', 'force_sell', 'emergency_sell']
+    :param current_time: datetime object, containing the current datetime
     :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
     :return bool: When True is returned, then the sell-order is placed on the exchange.
         False aborts the process